From a6981a0a50bd313e489bc507190757cf1064005c Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:13:33 -0700 Subject: [PATCH 1/4] refactor(vision): migrate ten classifier families Implement family-owned ImageToClassScores bindings and complete checkpoint-order logits. Preserve each graph, weight loader, image preprocessing policy, declared profile and original reference oracle. Exercise public C and C++ consumers through the existing family E2Es and CTest dependencies. Keep performance workloads and output contracts with family-owned references; no shared implementation or sibling dependency is added. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- families/timm_dpn/README.md | 66 +++++++ families/timm_dpn/model.py | 16 +- families/timm_dpn/runtime/CMakeLists.txt | 35 ++++ families/timm_dpn/runtime/pipeline.cpp | 80 +++++--- families/timm_dpn/runtime/pipeline.h | 20 +- families/timm_dpn/runtime/plugin.cpp | 17 +- families/timm_dpn/support.py | 4 +- .../timm_dpn/tests/cpp/test_task_contract.cpp | 175 +++++++++++++++++ .../tests/manifests/dpn68b-ra-in1k.json | 2 +- .../tests/manifests/dpn92-mx-in1k.json | 2 +- families/timm_dpn/tests/performance.yaml | 24 +++ .../timm_dpn/tests/performance_reference.py | 143 ++++++++++++++ families/timm_dpn/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_dpn/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_dpn/tests/test_e2e.py | 64 ++++++- .../tests/test_performance_reference.py | 90 +++++++++ families/timm_dpn/tests/test_sdk_metadata.py | 47 +++++ families/timm_hrnet/README.md | 66 +++++++ families/timm_hrnet/model.py | 16 +- families/timm_hrnet/runtime/CMakeLists.txt | 35 ++++ families/timm_hrnet/runtime/pipeline.cpp | 80 +++++--- families/timm_hrnet/runtime/pipeline.h | 23 ++- families/timm_hrnet/runtime/plugin.cpp | 17 +- families/timm_hrnet/support.py | 4 +- .../tests/cpp/test_task_contract.cpp | 176 ++++++++++++++++++ .../manifests/hrnet-w18-ms-aug-in1k.json | 2 +- families/timm_hrnet/tests/performance.yaml | 24 +++ .../timm_hrnet/tests/performance_reference.py | 143 ++++++++++++++ families/timm_hrnet/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_hrnet/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_hrnet/tests/test_e2e.py | 64 ++++++- .../tests/test_performance_reference.py | 90 +++++++++ .../timm_hrnet/tests/test_sdk_metadata.py | 47 +++++ families/timm_mnasnet/README.md | 66 +++++++ families/timm_mnasnet/model.py | 20 +- families/timm_mnasnet/runtime/CMakeLists.txt | 35 ++++ families/timm_mnasnet/runtime/pipeline.cpp | 53 ++++-- families/timm_mnasnet/runtime/pipeline.h | 23 ++- families/timm_mnasnet/runtime/plugin.cpp | 11 +- families/timm_mnasnet/support.py | 4 +- .../tests/cpp/test_task_contract.cpp | 163 ++++++++++++++++ .../manifests/mnasnet-100-rmsp-in1k.json | 2 +- families/timm_mnasnet/tests/performance.yaml | 24 +++ .../tests/performance_reference.py | 143 ++++++++++++++ families/timm_mnasnet/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_mnasnet/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_mnasnet/tests/test_e2e.py | 64 ++++++- families/timm_mnasnet/tests/test_model.py | 49 ++++- .../tests/test_performance_reference.py | 90 +++++++++ families/timm_nfnet/README.md | 66 +++++++ families/timm_nfnet/model.py | 16 +- families/timm_nfnet/runtime/CMakeLists.txt | 35 ++++ families/timm_nfnet/runtime/pipeline.cpp | 80 +++++--- families/timm_nfnet/runtime/pipeline.h | 23 ++- families/timm_nfnet/runtime/plugin.cpp | 17 +- families/timm_nfnet/support.py | 4 +- .../tests/cpp/test_task_contract.cpp | 176 ++++++++++++++++++ .../tests/manifests/dm-nfnet-f0-dm-in1k.json | 2 +- .../tests/manifests/dm-nfnet-f3-dm-in1k.json | 2 +- families/timm_nfnet/tests/performance.yaml | 24 +++ .../timm_nfnet/tests/performance_reference.py | 143 ++++++++++++++ families/timm_nfnet/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_nfnet/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_nfnet/tests/test_e2e.py | 64 ++++++- .../tests/test_performance_reference.py | 90 +++++++++ .../timm_nfnet/tests/test_sdk_metadata.py | 48 +++++ families/timm_repvgg/README.md | 66 +++++++ families/timm_repvgg/model.py | 16 +- families/timm_repvgg/runtime/CMakeLists.txt | 35 ++++ families/timm_repvgg/runtime/pipeline.cpp | 81 +++++--- families/timm_repvgg/runtime/pipeline.h | 25 ++- families/timm_repvgg/runtime/plugin.cpp | 17 +- families/timm_repvgg/support.py | 4 +- .../tests/cpp/test_task_contract.cpp | 176 ++++++++++++++++++ .../tests/manifests/repvgg-a2-rvgg-in1k.json | 2 +- families/timm_repvgg/tests/performance.yaml | 24 +++ .../tests/performance_reference.py | 143 ++++++++++++++ families/timm_repvgg/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_repvgg/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_repvgg/tests/test_e2e.py | 64 ++++++- families/timm_repvgg/tests/test_model.py | 4 +- .../tests/test_performance_reference.py | 90 +++++++++ .../timm_repvgg/tests/test_sdk_metadata.py | 47 +++++ families/timm_res2net/README.md | 66 +++++++ families/timm_res2net/model.py | 16 +- families/timm_res2net/runtime/CMakeLists.txt | 35 ++++ families/timm_res2net/runtime/pipeline.cpp | 81 +++++--- families/timm_res2net/runtime/pipeline.h | 25 ++- families/timm_res2net/runtime/plugin.cpp | 17 +- families/timm_res2net/support.py | 4 +- .../tests/cpp/test_task_contract.cpp | 176 ++++++++++++++++++ .../manifests/res2net50-26w-4s-in1k.json | 2 +- .../tests/manifests/res2net50d-in1k.json | 2 +- .../tests/manifests/res2next50-in1k.json | 2 +- families/timm_res2net/tests/performance.yaml | 24 +++ .../tests/performance_reference.py | 143 ++++++++++++++ families/timm_res2net/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_res2net/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_res2net/tests/test_e2e.py | 64 ++++++- .../tests/test_performance_reference.py | 90 +++++++++ .../timm_res2net/tests/test_sdk_metadata.py | 47 +++++ families/timm_senet/README.md | 66 +++++++ families/timm_senet/model.py | 16 +- families/timm_senet/runtime/CMakeLists.txt | 35 ++++ families/timm_senet/runtime/pipeline.cpp | 80 +++++--- families/timm_senet/runtime/pipeline.h | 23 ++- families/timm_senet/runtime/plugin.cpp | 17 +- families/timm_senet/support.py | 4 +- .../tests/cpp/test_task_contract.cpp | 176 ++++++++++++++++++ .../tests/manifests/senet154-gluon-in1k.json | 2 +- families/timm_senet/tests/performance.yaml | 24 +++ .../timm_senet/tests/performance_reference.py | 143 ++++++++++++++ families/timm_senet/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_senet/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_senet/tests/test_e2e.py | 64 ++++++- .../tests/test_performance_reference.py | 90 +++++++++ .../timm_senet/tests/test_sdk_metadata.py | 47 +++++ families/timm_seresnet/README.md | 66 +++++++ families/timm_seresnet/model.py | 16 +- families/timm_seresnet/runtime/CMakeLists.txt | 35 ++++ families/timm_seresnet/runtime/pipeline.cpp | 81 +++++--- families/timm_seresnet/runtime/pipeline.h | 25 ++- families/timm_seresnet/runtime/plugin.cpp | 19 +- families/timm_seresnet/support.py | 4 +- .../tests/cpp/test_task_contract.cpp | 176 ++++++++++++++++++ .../tests/manifests/seresnet50-a1-in1k.json | 2 +- families/timm_seresnet/tests/performance.yaml | 24 +++ .../tests/performance_reference.py | 143 ++++++++++++++ families/timm_seresnet/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_seresnet/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_seresnet/tests/test_e2e.py | 64 ++++++- .../tests/test_performance_reference.py | 90 +++++++++ .../timm_seresnet/tests/test_sdk_metadata.py | 47 +++++ families/timm_vgg/README.md | 66 +++++++ families/timm_vgg/model.py | 20 +- families/timm_vgg/runtime/CMakeLists.txt | 35 ++++ families/timm_vgg/runtime/pipeline.cpp | 52 ++++-- families/timm_vgg/runtime/pipeline.h | 20 +- families/timm_vgg/runtime/plugin.cpp | 11 +- families/timm_vgg/support.py | 4 +- .../timm_vgg/tests/cpp/test_task_contract.cpp | 163 ++++++++++++++++ .../tests/manifests/vgg16-tv-in1k.json | 2 +- families/timm_vgg/tests/performance.yaml | 24 +++ .../timm_vgg/tests/performance_reference.py | 143 ++++++++++++++ families/timm_vgg/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_vgg/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_vgg/tests/test_e2e.py | 64 ++++++- families/timm_vgg/tests/test_model.py | 49 ++++- .../tests/test_performance_reference.py | 90 +++++++++ families/timm_xception/README.md | 66 +++++++ families/timm_xception/model.py | 16 +- families/timm_xception/runtime/CMakeLists.txt | 35 ++++ families/timm_xception/runtime/pipeline.cpp | 81 +++++--- families/timm_xception/runtime/pipeline.h | 25 ++- families/timm_xception/runtime/plugin.cpp | 17 +- families/timm_xception/support.py | 4 +- .../tests/cpp/test_task_contract.cpp | 176 ++++++++++++++++++ .../tests/manifests/xception41-tf-in1k.json | 2 +- families/timm_xception/tests/performance.yaml | 24 +++ .../tests/performance_reference.py | 143 ++++++++++++++ families/timm_xception/tests/sdk_consumer.c | 166 +++++++++++++++++ families/timm_xception/tests/sdk_consumer.cpp | 110 +++++++++++ families/timm_xception/tests/test_e2e.py | 64 ++++++- .../tests/test_performance_reference.py | 90 +++++++++ .../timm_xception/tests/test_sdk_metadata.py | 47 +++++ 165 files changed, 10137 insertions(+), 432 deletions(-) create mode 100644 families/timm_dpn/README.md create mode 100644 families/timm_dpn/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_dpn/tests/performance.yaml create mode 100644 families/timm_dpn/tests/performance_reference.py create mode 100644 families/timm_dpn/tests/sdk_consumer.c create mode 100644 families/timm_dpn/tests/sdk_consumer.cpp create mode 100644 families/timm_dpn/tests/test_performance_reference.py create mode 100644 families/timm_dpn/tests/test_sdk_metadata.py create mode 100644 families/timm_hrnet/README.md create mode 100644 families/timm_hrnet/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_hrnet/tests/performance.yaml create mode 100644 families/timm_hrnet/tests/performance_reference.py create mode 100644 families/timm_hrnet/tests/sdk_consumer.c create mode 100644 families/timm_hrnet/tests/sdk_consumer.cpp create mode 100644 families/timm_hrnet/tests/test_performance_reference.py create mode 100644 families/timm_hrnet/tests/test_sdk_metadata.py create mode 100644 families/timm_mnasnet/README.md create mode 100644 families/timm_mnasnet/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_mnasnet/tests/performance.yaml create mode 100644 families/timm_mnasnet/tests/performance_reference.py create mode 100644 families/timm_mnasnet/tests/sdk_consumer.c create mode 100644 families/timm_mnasnet/tests/sdk_consumer.cpp create mode 100644 families/timm_mnasnet/tests/test_performance_reference.py create mode 100644 families/timm_nfnet/README.md create mode 100644 families/timm_nfnet/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_nfnet/tests/performance.yaml create mode 100644 families/timm_nfnet/tests/performance_reference.py create mode 100644 families/timm_nfnet/tests/sdk_consumer.c create mode 100644 families/timm_nfnet/tests/sdk_consumer.cpp create mode 100644 families/timm_nfnet/tests/test_performance_reference.py create mode 100644 families/timm_nfnet/tests/test_sdk_metadata.py create mode 100644 families/timm_repvgg/README.md create mode 100644 families/timm_repvgg/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_repvgg/tests/performance.yaml create mode 100644 families/timm_repvgg/tests/performance_reference.py create mode 100644 families/timm_repvgg/tests/sdk_consumer.c create mode 100644 families/timm_repvgg/tests/sdk_consumer.cpp create mode 100644 families/timm_repvgg/tests/test_performance_reference.py create mode 100644 families/timm_repvgg/tests/test_sdk_metadata.py create mode 100644 families/timm_res2net/README.md create mode 100644 families/timm_res2net/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_res2net/tests/performance.yaml create mode 100644 families/timm_res2net/tests/performance_reference.py create mode 100644 families/timm_res2net/tests/sdk_consumer.c create mode 100644 families/timm_res2net/tests/sdk_consumer.cpp create mode 100644 families/timm_res2net/tests/test_performance_reference.py create mode 100644 families/timm_res2net/tests/test_sdk_metadata.py create mode 100644 families/timm_senet/README.md create mode 100644 families/timm_senet/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_senet/tests/performance.yaml create mode 100644 families/timm_senet/tests/performance_reference.py create mode 100644 families/timm_senet/tests/sdk_consumer.c create mode 100644 families/timm_senet/tests/sdk_consumer.cpp create mode 100644 families/timm_senet/tests/test_performance_reference.py create mode 100644 families/timm_senet/tests/test_sdk_metadata.py create mode 100644 families/timm_seresnet/README.md create mode 100644 families/timm_seresnet/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_seresnet/tests/performance.yaml create mode 100644 families/timm_seresnet/tests/performance_reference.py create mode 100644 families/timm_seresnet/tests/sdk_consumer.c create mode 100644 families/timm_seresnet/tests/sdk_consumer.cpp create mode 100644 families/timm_seresnet/tests/test_performance_reference.py create mode 100644 families/timm_seresnet/tests/test_sdk_metadata.py create mode 100644 families/timm_vgg/README.md create mode 100644 families/timm_vgg/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_vgg/tests/performance.yaml create mode 100644 families/timm_vgg/tests/performance_reference.py create mode 100644 families/timm_vgg/tests/sdk_consumer.c create mode 100644 families/timm_vgg/tests/sdk_consumer.cpp create mode 100644 families/timm_vgg/tests/test_performance_reference.py create mode 100644 families/timm_xception/README.md create mode 100644 families/timm_xception/tests/cpp/test_task_contract.cpp create mode 100644 families/timm_xception/tests/performance.yaml create mode 100644 families/timm_xception/tests/performance_reference.py create mode 100644 families/timm_xception/tests/sdk_consumer.c create mode 100644 families/timm_xception/tests/sdk_consumer.cpp create mode 100644 families/timm_xception/tests/test_performance_reference.py create mode 100644 families/timm_xception/tests/test_sdk_metadata.py diff --git a/families/timm_dpn/README.md b/families/timm_dpn/README.md new file mode 100644 index 0000000000..adcab83d61 --- /dev/null +++ b/families/timm_dpn/README.md @@ -0,0 +1,66 @@ +# timm DPN + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/dpn68b.ra_in1k -o model.bundle +trtmc classify model.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_dpn_task_contract test_timm_dpn_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_dpn_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_dpn_sdk_c model.bundle build image.rgb.f32 480 640 +build/test_timm_dpn_sdk_cpp model.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_dpn.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_dpn/model.py b/families/timm_dpn/model.py index 187e92f12a..dc52d7c42a 100644 --- a/families/timm_dpn/model.py +++ b/families/timm_dpn/model.py @@ -458,8 +458,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_dpn does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_dpn does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_dpn supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_dpn supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_dpn does not support quantization") if request.fp32_layers: @@ -474,11 +474,23 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: str(request.precision).lower(), bool(request.verbose), ) + vocabulary_id = raw.get("vocabulary_id", "") + labels = raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm DPN vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != runtime["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm DPN label_names must name every class") writer.set_header(family="timm_dpn", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( "runtime.json", { + "num_classes": runtime["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "input_image_h": runtime["image_height"], "input_image_w": runtime["image_width"], "crop_pct": runtime["crop_pct"], diff --git a/families/timm_dpn/runtime/CMakeLists.txt b/families/timm_dpn/runtime/CMakeLists.txt index 819fa39425..5caf774680 100644 --- a/families/timm_dpn/runtime/CMakeLists.txt +++ b/families/timm_dpn/runtime/CMakeLists.txt @@ -35,6 +35,41 @@ install(TARGETS trtmc_model_timm_dpn ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_dpn_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_dpn/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_dpn_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_dpn_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_dpn_task_contract PRIVATE + trtmc_model_timm_dpn trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_dpn_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_dpn_task_contract COMMAND test_timm_dpn_task_contract) + set_tests_properties(timm_dpn_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_dpn_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_dpn/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_dpn_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_dpn_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_dpn_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_dpn_task_contract + test_timm_dpn_sdk_c test_timm_dpn_sdk_cpp + ) + add_executable(test_timm_dpn_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_dpn/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_dpn/runtime/pipeline.cpp b/families/timm_dpn/runtime/pipeline.cpp index 8c29470fee..74894a2c0b 100644 --- a/families/timm_dpn/runtime/pipeline.cpp +++ b/families/timm_dpn/runtime/pipeline.cpp @@ -5,50 +5,78 @@ #include "families/timm_dpn/runtime/pipeline.h" -#include #include +#include #include #include namespace trtmc { + namespace { -const Tensor* find_logits(const TensorMap& outputs) { +const Tensor& require_logits(const TensorMap& outputs) { for (const auto& [name, tensor] : outputs) { - if (name.find("logits") != std::string::npos || outputs.size() == 1) - return &tensor; + if (name.find("logits") == std::string::npos && outputs.size() != 1) + continue; + if (tensor.data == nullptr || tensor.dtype != DType::kFloat32 || tensor.numel() <= 0) + throw std::runtime_error("timm DPN engine must return nonempty float32 logits"); + return tensor; } - return nullptr; + throw std::runtime_error("timm DPN engine did not return logits"); } } // namespace TimmDpnImageClassificationPipeline::TimmDpnImageClassificationPipeline( - std::unique_ptr model, TimmDpnPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmDpnPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmDpnImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm DPN class metadata does not match its output size"); } -ClassificationResult TimmDpnImageClassificationPipeline::classify(const float* pixels, - int32_t height, int32_t width) { - auto values = preprocess_timm_dpn_image(pixels, height, width, preprocess_config_); - Tensor input; - input.data = values.data(); - input.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; - input.dtype = DType::kFloat32; - const auto outputs = model_->forward({{"pixel_values", input}}); - const Tensor* logits = find_logits(outputs); - if (logits == nullptr || logits->numel() <= 0) - throw std::runtime_error("timm DPN engine returned no logits"); - if (logits->dtype != DType::kFloat32) - throw std::runtime_error("timm DPN logits must be float32"); - ClassificationResult result; - result.logits.resize(static_cast(logits->numel())); - std::memcpy(result.logits.data(), logits->data, result.logits.size() * sizeof(float)); - const auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = *best; +internal::LabelScoresResult +TimmDpnImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm DPN has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm DPN requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_dpn_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); + + Tensor img_t; + img_t.data = pixel_values.data(); + img_t.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; + img_t.dtype = DType::kFloat32; + + auto outputs = model_->forward({{"pixel_values", img_t}}); + internal::LabelScoresResult result; + + const auto& logits_tensor = require_logits(outputs); + const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm DPN logits do not match its configured class count"); + + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, + static_cast(n) * sizeof(float)); + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_dpn/runtime/pipeline.h b/families/timm_dpn/runtime/pipeline.h index 5ffc881960..1423a8d6a5 100644 --- a/families/timm_dpn/runtime/pipeline.h +++ b/families/timm_dpn/runtime/pipeline.h @@ -6,23 +6,35 @@ #pragma once #include "families/timm_dpn/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmDpnImageClassificationPipeline final : public IImageClassification { +class TimmDpnImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: explicit TimmDpnImageClassificationPipeline(std::unique_ptr model, - TimmDpnPreprocessConfig preprocess_config = {}); + TimmDpnPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, + std::vector labels); - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmDpnPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_dpn/runtime/plugin.cpp b/families/timm_dpn/runtime/plugin.cpp index 44c9b8d31b..3b7a6e9cdb 100644 --- a/families/timm_dpn/runtime/plugin.cpp +++ b/families/timm_dpn/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmDpnPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmDpnPreprocessConfig parse_config(const nlohmann::json& json) { TimmDpnPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -34,7 +33,7 @@ TimmDpnPreprocessConfig parse_config(const std::vector& data) { if (config.input_image_h <= 0 || config.input_image_w <= 0 || config.crop_pct <= 0.0F || config.crop_pct > 1.0F || config.image_mean.size() != 3 || config.image_std.size() != 3 || (config.interpolation != "bilinear" && config.interpolation != "bicubic")) { - throw std::runtime_error("timm DPN runtime.json does not match its contract"); + throw std::runtime_error("timm DPN runtime.json does not match its runtime contract"); } return config; } @@ -53,9 +52,13 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_dpn/support.py b/families/timm_dpn/support.py index 310ae5e287..94ff369ffe 100644 --- a/families/timm_dpn/support.py +++ b/families/timm_dpn/support.py @@ -9,6 +9,6 @@ describe = family_support( model_types=("timm_dpn", "dpn68", "dpn68b", "dpn92", "dpn98", "dpn107", "dpn131"), architectures=("dpn68", "dpn68b", "dpn92", "dpn98", "dpn107", "dpn131"), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_dpn/tests/cpp/test_task_contract.cpp b/families/timm_dpn/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..1d761aa9d0 --- /dev/null +++ b/families/timm_dpn/tests/cpp/test_task_contract.cpp @@ -0,0 +1,175 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_dpn/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{output_name, {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + std::string output_name{"logits"}; + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmDpnPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmDpnImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmDpnImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +void test_existing_named_output() { + auto module = std::make_unique(); + module->output_name = "class_scores"; + const auto expected = module->logits; + trtmc::TimmDpnImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", {}); + const std::vector pixels(12, 0.75F); + require(model.run(request(pixels), {}).scores == expected, + "a single named classifier output retains the existing runtime contract"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + test_existing_named_output(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_dpn/tests/manifests/dpn68b-ra-in1k.json b/families/timm_dpn/tests/manifests/dpn68b-ra-in1k.json index 93b23f0d19..24c2e26089 100644 --- a/families/timm_dpn/tests/manifests/dpn68b-ra-in1k.json +++ b/families/timm_dpn/tests/manifests/dpn68b-ra-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "fb49a617159bd2a0d6099742a6c57ef2d884dad9", "bundle": "dpn68b-ra-in1k.bundle", "family": "timm_dpn", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_dpn/tests/manifests/dpn92-mx-in1k.json b/families/timm_dpn/tests/manifests/dpn92-mx-in1k.json index 4b4176a9f7..294062745d 100644 --- a/families/timm_dpn/tests/manifests/dpn92-mx-in1k.json +++ b/families/timm_dpn/tests/manifests/dpn92-mx-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "8f2a71d2525254e14b0830a97250d841d5c4cae8", "bundle": "dpn92-mx-in1k.bundle", "family": "timm_dpn", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_dpn/tests/performance.yaml b/families/timm_dpn/tests/performance.yaml new file mode 100644 index 0000000000..5c091cf48b --- /dev/null +++ b/families/timm_dpn/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-dpn-performance +entries: + - id: timm_dpn.classify + family: timm_dpn + operation: classify + model: dpn68b-ra-in1k + workload: + testcase: dpn68b-ra-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_dpn/tests/performance_reference.py b/families/timm_dpn/tests/performance_reference.py new file mode 100644 index 0000000000..cd68cb9b23 --- /dev/null +++ b/families/timm_dpn/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time DPN inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model( + config["architecture"], pretrained=False, + pretrained_cfg=config["pretrained_cfg"], num_classes=int(config["num_classes"]), + checkpoint_path=str(checkpoint / "model.safetensors"), + ) + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("DPN must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("DPN returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_dpn" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_dpn image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_dpn/tests/sdk_consumer.c b/families/timm_dpn/tests/sdk_consumer.c new file mode 100644 index 0000000000..d91bb7528d --- /dev/null +++ b/families/timm_dpn/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "DPN must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "DPN must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_dpn/tests/sdk_consumer.cpp b/families/timm_dpn/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..614b606eac --- /dev/null +++ b/families/timm_dpn/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("DPN must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("DPN must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_dpn/tests/test_e2e.py b/families/timm_dpn/tests/test_e2e.py index 64d4a1ed1a..93b54519ca 100644 --- a/families/timm_dpn/tests/test_e2e.py +++ b/families/timm_dpn/tests/test_e2e.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import math import os import subprocess from pathlib import Path @@ -27,7 +28,7 @@ def _cases() -> dict[str, tuple[dict, dict]]: for path in sorted(MANIFEST_ROOT.glob("*.json")): manifest = json.loads(path.read_text(encoding="utf-8")) assert manifest["family"] == FAMILY - assert manifest["task"] == "classification" + assert manifest["task"] == "image_to_class_scores" for case in manifest["testcases"]: name = str(case["name"]) assert name not in result @@ -104,6 +105,58 @@ def _asset(case: dict) -> Path: return path +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case)).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_dpn_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: manifest, case = CASES[case_name] record_evidence("inputs", {"manifest": manifest, "case": case}) @@ -175,3 +228,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: record_evidence("reference", {"top_class": int(np.argmax(expected)), "logits": expected}) with evidence_stage("compare"): assert int(actual["top_class"]) == int(np.argmax(expected)) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert len(actual["scores"]) == int(config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, int(np.argmax(expected)), tmp_path) diff --git a/families/timm_dpn/tests/test_performance_reference.py b/families/timm_dpn/tests/test_performance_reference.py new file mode 100644 index 0000000000..38cb3aa06d --- /dev/null +++ b/families/timm_dpn/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_dpn.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_dpn", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_dpn.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_dpn/tests/test_sdk_metadata.py b/families/timm_dpn/tests/test_sdk_metadata.py new file mode 100644 index 0000000000..50c1c7fa3e --- /dev/null +++ b/families/timm_dpn/tests/test_sdk_metadata.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The builder publishes complete checkpoint-owned Task metadata.""" + +from types import SimpleNamespace + +import pytest + +from families.timm_dpn import model +from tensorrt_model_connect import BuildRequest + + +@pytest.mark.parametrize("metadata,invalid", [ + ({}, False), + ({"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, False), + ({"label_names": ["only-one"]}, True), + ({"label_names": ["a", "b", "", "d", "e"]}, True), + ({"label_names": 5}, True), + ({"vocabulary_id": 5}, True), +]) +def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): + raw = {"num_classes": 5, **metadata} + runtime = {"image_height": 2, "image_width": 2, "num_classes": 5, + "crop_pct": 1.0, "interpolation": "bilinear", + "mean": [0.5] * 3, "std": [0.25] * 3} + monkeypatch.setattr(model, "_read_config", lambda _: raw) + monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) + monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + sections = {} + headers = [] + writer = SimpleNamespace(set_header=lambda **value: headers.append(value), + add_bytes=lambda key, value: sections.update({key: value}), + add_json=lambda key, value: sections.update({key: value})) + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_dpn", task="image_to_class_scores", precision="fp32") + if invalid: + with pytest.raises(ValueError, match="vocabulary_id|label_names"): + model.build(request, writer) + assert not sections and not headers + return + model.build(request, writer) + assert headers == [{"family": "timm_dpn", "task": "image_to_class_scores", "backend": "trt"}] + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) diff --git a/families/timm_hrnet/README.md b/families/timm_hrnet/README.md new file mode 100644 index 0000000000..e681a46d2e --- /dev/null +++ b/families/timm_hrnet/README.md @@ -0,0 +1,66 @@ +# timm HRNet + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/hrnet_w18.ms_aug_in1k -o model.bundle +trtmc classify model.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_hrnet_task_contract test_timm_hrnet_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_hrnet_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_hrnet_sdk_c model.bundle build image.rgb.f32 480 640 +build/test_timm_hrnet_sdk_cpp model.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_hrnet.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_hrnet/model.py b/families/timm_hrnet/model.py index cd8f6d8883..ecd9645fe6 100644 --- a/families/timm_hrnet/model.py +++ b/families/timm_hrnet/model.py @@ -462,8 +462,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_hrnet does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_hrnet does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_hrnet supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_hrnet supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_hrnet does not support quantization") if request.fp32_layers: @@ -477,11 +477,23 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: str(request.precision).lower(), bool(request.verbose), ) + vocabulary_id = raw.get("vocabulary_id", "") + labels = raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm HRNet vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != runtime["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm HRNet label_names must name every class") writer.set_header(family="timm_hrnet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( "runtime.json", { + "num_classes": runtime["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "input_image_h": runtime["image_height"], "input_image_w": runtime["image_width"], "crop_pct": runtime["crop_pct"], diff --git a/families/timm_hrnet/runtime/CMakeLists.txt b/families/timm_hrnet/runtime/CMakeLists.txt index 254e57f585..bec069d410 100644 --- a/families/timm_hrnet/runtime/CMakeLists.txt +++ b/families/timm_hrnet/runtime/CMakeLists.txt @@ -35,6 +35,41 @@ install(TARGETS trtmc_model_timm_hrnet ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_hrnet_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_hrnet/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_hrnet_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_hrnet_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_hrnet_task_contract PRIVATE + trtmc_model_timm_hrnet trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_hrnet_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_hrnet_task_contract COMMAND test_timm_hrnet_task_contract) + set_tests_properties(timm_hrnet_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_hrnet_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_hrnet/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_hrnet_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_hrnet_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_hrnet_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_hrnet_task_contract + test_timm_hrnet_sdk_c test_timm_hrnet_sdk_cpp + ) + add_executable(test_timm_hrnet_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_hrnet/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_hrnet/runtime/pipeline.cpp b/families/timm_hrnet/runtime/pipeline.cpp index 7e1ba20725..6a02e24412 100644 --- a/families/timm_hrnet/runtime/pipeline.cpp +++ b/families/timm_hrnet/runtime/pipeline.cpp @@ -5,50 +5,78 @@ #include "families/timm_hrnet/runtime/pipeline.h" -#include #include +#include #include #include namespace trtmc { + namespace { -const Tensor* find_logits(const TensorMap& outputs) { +const Tensor& require_logits(const TensorMap& outputs) { for (const auto& [name, tensor] : outputs) { - if (name.find("logits") != std::string::npos || outputs.size() == 1) - return &tensor; + if (name.find("logits") == std::string::npos && outputs.size() != 1) + continue; + if (tensor.data == nullptr || tensor.dtype != DType::kFloat32 || tensor.numel() <= 0) + throw std::runtime_error("timm HRNet engine must return nonempty float32 logits"); + return tensor; } - return nullptr; + throw std::runtime_error("timm HRNet engine did not return logits"); } } // namespace TimmHRNetImageClassificationPipeline::TimmHRNetImageClassificationPipeline( - std::unique_ptr model, TimmHRNetPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmHRNetPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmHRNetImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm HRNet class metadata does not match its output size"); } -ClassificationResult TimmHRNetImageClassificationPipeline::classify(const float* pixels, - int32_t height, int32_t width) { - auto values = preprocess_timm_hrnet_image(pixels, height, width, preprocess_config_); - Tensor input; - input.data = values.data(); - input.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; - input.dtype = DType::kFloat32; - const auto outputs = model_->forward({{"pixel_values", input}}); - const Tensor* logits = find_logits(outputs); - if (logits == nullptr || logits->numel() <= 0) - throw std::runtime_error("timm HRNet engine returned no logits"); - if (logits->dtype != DType::kFloat32) - throw std::runtime_error("timm HRNet logits must be float32"); - ClassificationResult result; - result.logits.resize(static_cast(logits->numel())); - std::memcpy(result.logits.data(), logits->data, result.logits.size() * sizeof(float)); - const auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = *best; +internal::LabelScoresResult +TimmHRNetImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm HRNet has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm HRNet requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_hrnet_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); + + Tensor img_t; + img_t.data = pixel_values.data(); + img_t.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; + img_t.dtype = DType::kFloat32; + + auto outputs = model_->forward({{"pixel_values", img_t}}); + internal::LabelScoresResult result; + + const auto& logits_tensor = require_logits(outputs); + const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm HRNet logits do not match its configured class count"); + + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, + static_cast(n) * sizeof(float)); + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_hrnet/runtime/pipeline.h b/families/timm_hrnet/runtime/pipeline.h index fc56537913..54004af3f3 100644 --- a/families/timm_hrnet/runtime/pipeline.h +++ b/families/timm_hrnet/runtime/pipeline.h @@ -6,23 +6,36 @@ #pragma once #include "families/timm_hrnet/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmHRNetImageClassificationPipeline final : public IImageClassification { +class TimmHRNetImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: explicit TimmHRNetImageClassificationPipeline(std::unique_ptr model, - TimmHRNetPreprocessConfig preprocess_config = {}); - - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + TimmHRNetPreprocessConfig preprocess_config, + std::int32_t num_classes, + std::string vocabulary_id, + std::vector labels); + + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmHRNetPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_hrnet/runtime/plugin.cpp b/families/timm_hrnet/runtime/plugin.cpp index bdaf9f8a66..fd7ab27f5b 100644 --- a/families/timm_hrnet/runtime/plugin.cpp +++ b/families/timm_hrnet/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmHRNetPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmHRNetPreprocessConfig parse_config(const nlohmann::json& json) { TimmHRNetPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -34,7 +33,7 @@ TimmHRNetPreprocessConfig parse_config(const std::vector& data) { if (config.input_image_h <= 0 || config.input_image_w <= 0 || config.crop_pct <= 0.0F || config.crop_pct > 1.0F || config.image_mean.size() != 3 || config.image_std.size() != 3 || (config.interpolation != "bilinear" && config.interpolation != "bicubic")) { - throw std::runtime_error("timm HRNet runtime.json does not match its contract"); + throw std::runtime_error("timm HRNet runtime.json does not match its runtime contract"); } return config; } @@ -53,9 +52,13 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_hrnet/support.py b/families/timm_hrnet/support.py index 6ee801bfb8..b3cf09d752 100644 --- a/families/timm_hrnet/support.py +++ b/families/timm_hrnet/support.py @@ -9,6 +9,6 @@ describe = family_support( model_types=("timm_hrnet", "hrnet_w18"), architectures=("hrnet_w18",), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_hrnet/tests/cpp/test_task_contract.cpp b/families/timm_hrnet/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..812ed5d85a --- /dev/null +++ b/families/timm_hrnet/tests/cpp/test_task_contract.cpp @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_hrnet/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{output_name, {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + std::string output_name{"logits"}; + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmHRNetPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmHRNetImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmHRNetImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +void test_existing_named_output() { + auto module = std::make_unique(); + module->output_name = "class_scores"; + const auto expected = module->logits; + trtmc::TimmHRNetImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const std::vector pixels(12, 0.75F); + require(model.run(request(pixels), {}).scores == expected, + "a single named classifier output retains the existing runtime contract"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + test_existing_named_output(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_hrnet/tests/manifests/hrnet-w18-ms-aug-in1k.json b/families/timm_hrnet/tests/manifests/hrnet-w18-ms-aug-in1k.json index 1155df70cc..95c7dedadb 100644 --- a/families/timm_hrnet/tests/manifests/hrnet-w18-ms-aug-in1k.json +++ b/families/timm_hrnet/tests/manifests/hrnet-w18-ms-aug-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "7e2c5583769f54514fd87e3ba9de408e33eaba0f", "bundle": "hrnet-w18-ms-aug-in1k.bundle", "family": "timm_hrnet", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_hrnet/tests/performance.yaml b/families/timm_hrnet/tests/performance.yaml new file mode 100644 index 0000000000..988eef9192 --- /dev/null +++ b/families/timm_hrnet/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-hrnet-performance +entries: + - id: timm_hrnet.classify + family: timm_hrnet + operation: classify + model: hrnet-w18-ms-aug-in1k + workload: + testcase: hrnet-w18-ms-aug-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_hrnet/tests/performance_reference.py b/families/timm_hrnet/tests/performance_reference.py new file mode 100644 index 0000000000..351ab68df3 --- /dev/null +++ b/families/timm_hrnet/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time HRNet inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model( + config["architecture"], pretrained=False, + pretrained_cfg=config["pretrained_cfg"], num_classes=int(config["num_classes"]), + checkpoint_path=str(checkpoint / "model.safetensors"), + ) + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("HRNet must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("HRNet returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_hrnet" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_hrnet image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_hrnet/tests/sdk_consumer.c b/families/timm_hrnet/tests/sdk_consumer.c new file mode 100644 index 0000000000..d70183da2a --- /dev/null +++ b/families/timm_hrnet/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "HRNet must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "HRNet must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_hrnet/tests/sdk_consumer.cpp b/families/timm_hrnet/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..30406660b4 --- /dev/null +++ b/families/timm_hrnet/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("HRNet must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("HRNet must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_hrnet/tests/test_e2e.py b/families/timm_hrnet/tests/test_e2e.py index beaa1beb43..483d07eaff 100644 --- a/families/timm_hrnet/tests/test_e2e.py +++ b/families/timm_hrnet/tests/test_e2e.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import math import os import subprocess from pathlib import Path @@ -27,7 +28,7 @@ def _cases() -> dict[str, tuple[dict, dict]]: for path in sorted(MANIFEST_ROOT.glob("*.json")): manifest = json.loads(path.read_text(encoding="utf-8")) assert manifest["family"] == FAMILY - assert manifest["task"] == "classification" + assert manifest["task"] == "image_to_class_scores" for case in manifest["testcases"]: name = str(case["name"]) assert name not in result @@ -104,6 +105,58 @@ def _asset(case: dict) -> Path: return path +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case)).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_hrnet_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: manifest, case = CASES[case_name] record_evidence("inputs", {"manifest": manifest, "case": case}) @@ -175,3 +228,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: record_evidence("reference", {"top_class": int(np.argmax(expected)), "logits": expected}) with evidence_stage("compare"): assert int(actual["top_class"]) == int(np.argmax(expected)) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert len(actual["scores"]) == int(config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, int(np.argmax(expected)), tmp_path) diff --git a/families/timm_hrnet/tests/test_performance_reference.py b/families/timm_hrnet/tests/test_performance_reference.py new file mode 100644 index 0000000000..c239e3748a --- /dev/null +++ b/families/timm_hrnet/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_hrnet.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_hrnet", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_hrnet.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_hrnet/tests/test_sdk_metadata.py b/families/timm_hrnet/tests/test_sdk_metadata.py new file mode 100644 index 0000000000..32b10b7598 --- /dev/null +++ b/families/timm_hrnet/tests/test_sdk_metadata.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The builder publishes complete checkpoint-owned Task metadata.""" + +from types import SimpleNamespace + +import pytest + +from families.timm_hrnet import model +from tensorrt_model_connect import BuildRequest + + +@pytest.mark.parametrize("metadata,invalid", [ + ({}, False), + ({"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, False), + ({"label_names": ["only-one"]}, True), + ({"label_names": ["a", "b", "", "d", "e"]}, True), + ({"label_names": 5}, True), + ({"vocabulary_id": 5}, True), +]) +def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): + raw = {"num_classes": 5, **metadata} + runtime = {"image_height": 2, "image_width": 2, "num_classes": 5, + "crop_pct": 1.0, "interpolation": "bilinear", + "mean": [0.5] * 3, "std": [0.25] * 3} + monkeypatch.setattr(model, "_read_config", lambda _: raw) + monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) + monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + sections = {} + headers = [] + writer = SimpleNamespace(set_header=lambda **value: headers.append(value), + add_bytes=lambda key, value: sections.update({key: value}), + add_json=lambda key, value: sections.update({key: value})) + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_hrnet", task="image_to_class_scores", precision="fp32") + if invalid: + with pytest.raises(ValueError, match="vocabulary_id|label_names"): + model.build(request, writer) + assert not sections and not headers + return + model.build(request, writer) + assert headers == [{"family": "timm_hrnet", "task": "image_to_class_scores", "backend": "trt"}] + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) diff --git a/families/timm_mnasnet/README.md b/families/timm_mnasnet/README.md new file mode 100644 index 0000000000..7387b83834 --- /dev/null +++ b/families/timm_mnasnet/README.md @@ -0,0 +1,66 @@ +# timm MNASNet + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/mnasnet_100.rmsp_in1k -o mnasnet.bundle +trtmc classify mnasnet.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_mnasnet_task_contract test_timm_mnasnet_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_mnasnet_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_mnasnet_sdk_c mnasnet.bundle build image.rgb.f32 480 640 +build/test_timm_mnasnet_sdk_cpp mnasnet.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_mnasnet.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_mnasnet/model.py b/families/timm_mnasnet/model.py index e6791593d3..21366deda5 100644 --- a/families/timm_mnasnet/model.py +++ b/families/timm_mnasnet/model.py @@ -392,10 +392,21 @@ def get_bundle_config_overrides(self, config: ModelConfig) -> dict: cfg = config.raw.get("_timm_mnasnet_config") if cfg is None: raise RuntimeError("load_weights must run before reading bundle config") + vocabulary_id = config.raw.get("vocabulary_id", "") + labels = config.raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm MNASNet vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != cfg["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm MNASNet label_names must name every class") return { "input_image_h": cfg["image_size_h"], "input_image_w": cfg["image_size_w"], "num_classes": cfg["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "image_mean": cfg["mean"], "image_std": cfg["std"], "crop_pct": cfg["crop_pct"], @@ -420,8 +431,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_mnasnet does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_mnasnet does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_mnasnet supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_mnasnet supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_mnasnet does not support quantization") if request.fp32_layers: @@ -442,14 +453,17 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: precision=precision, verbose=bool(request.verbose), ) + runtime_source = model.get_bundle_config_overrides(config) writer.set_header(family="timm_mnasnet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) - runtime_source = model.get_bundle_config_overrides(config) writer.add_json( "runtime.json", { key: runtime_source[key] for key in ( + "num_classes", + "vocabulary_id", + "labels", "input_image_h", "input_image_w", "crop_pct", diff --git a/families/timm_mnasnet/runtime/CMakeLists.txt b/families/timm_mnasnet/runtime/CMakeLists.txt index b0fe28a03b..2b7adc530e 100644 --- a/families/timm_mnasnet/runtime/CMakeLists.txt +++ b/families/timm_mnasnet/runtime/CMakeLists.txt @@ -38,6 +38,41 @@ install(TARGETS trtmc_model_timm_mnasnet ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_mnasnet_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_mnasnet/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_mnasnet_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_mnasnet_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_mnasnet_task_contract PRIVATE + trtmc_model_timm_mnasnet trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_mnasnet_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_mnasnet_task_contract COMMAND test_timm_mnasnet_task_contract) + set_tests_properties(timm_mnasnet_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_mnasnet_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_mnasnet/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_mnasnet_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_mnasnet_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_mnasnet_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_mnasnet_task_contract + test_timm_mnasnet_sdk_c test_timm_mnasnet_sdk_cpp + ) + add_executable(test_timm_mnasnet_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_mnasnet/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_mnasnet/runtime/pipeline.cpp b/families/timm_mnasnet/runtime/pipeline.cpp index 9ae3ee3aa3..4dc52731c3 100644 --- a/families/timm_mnasnet/runtime/pipeline.cpp +++ b/families/timm_mnasnet/runtime/pipeline.cpp @@ -5,8 +5,8 @@ #include "families/timm_mnasnet/runtime/pipeline.h" -#include #include +#include #include #include @@ -18,24 +18,44 @@ const Tensor& require_logits(const TensorMap& outputs) { const auto found = outputs.find("logits"); if (found == outputs.end()) throw std::runtime_error("timm MNASNet engine did not return logits"); - if (found->second.numel() <= 0) - throw std::runtime_error("timm MNASNet engine returned empty logits"); + if (found->second.data == nullptr || found->second.dtype != DType::kFloat32 || + found->second.numel() <= 0) + throw std::runtime_error("timm MNASNet engine must return nonempty float32 logits"); return found->second; } } // namespace TimmMnasnetImageClassificationPipeline::TimmMnasnetImageClassificationPipeline( - std::unique_ptr model, TimmMnasnetPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmMnasnetPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmMnasnetImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm MNASNet class metadata does not match its output size"); } -ClassificationResult TimmMnasnetImageClassificationPipeline::classify(const float* pixels, - int32_t height, - int32_t width) { - auto pixel_values = preprocess_timm_mnasnet_image(pixels, height, width, preprocess_config_); +internal::LabelScoresResult +TimmMnasnetImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm MNASNet has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm MNASNet requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_mnasnet_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); Tensor img_t; img_t.data = pixel_values.data(); @@ -43,18 +63,19 @@ ClassificationResult TimmMnasnetImageClassificationPipeline::classify(const floa img_t.dtype = DType::kFloat32; auto outputs = model_->forward({{"pixel_values", img_t}}); - ClassificationResult result; + internal::LabelScoresResult result; const auto& logits_tensor = require_logits(outputs); const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm MNASNet logits do not match its configured class count"); - result.logits.resize(static_cast(n)); - std::memcpy(result.logits.data(), logits_tensor.data, + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, static_cast(n) * sizeof(float)); - - auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = (best == result.logits.end()) ? 0.0F : *best; + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_mnasnet/runtime/pipeline.h b/families/timm_mnasnet/runtime/pipeline.h index aac7271b96..e21b028466 100644 --- a/families/timm_mnasnet/runtime/pipeline.h +++ b/families/timm_mnasnet/runtime/pipeline.h @@ -6,23 +6,36 @@ #pragma once #include "families/timm_mnasnet/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmMnasnetImageClassificationPipeline final : public IImageClassification { +class TimmMnasnetImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: explicit TimmMnasnetImageClassificationPipeline(std::unique_ptr model, - TimmMnasnetPreprocessConfig preprocess_config); - - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + TimmMnasnetPreprocessConfig preprocess_config, + std::int32_t num_classes, + std::string vocabulary_id, + std::vector labels); + + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmMnasnetPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_mnasnet/runtime/plugin.cpp b/families/timm_mnasnet/runtime/plugin.cpp index 7a09bfcec2..16ad3ef1dc 100644 --- a/families/timm_mnasnet/runtime/plugin.cpp +++ b/families/timm_mnasnet/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmMnasnetPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmMnasnetPreprocessConfig parse_config(const nlohmann::json& json) { TimmMnasnetPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -55,7 +54,11 @@ extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context throw std::invalid_argument("timm_mnasnet does not support --kv-cache-size"); const auto& config_data = trtmc::timm_mnasnet::require_section(context.reader, "runtime.json"); const auto& plan = trtmc::timm_mnasnet::require_section(context.reader, "engine.plan"); - auto config = trtmc::timm_mnasnet::parse_config(config_data); + const auto metadata = nlohmann::json::parse(config_data.begin(), config_data.end()); + auto config = trtmc::timm_mnasnet::parse_config(metadata); auto engine = trtmc::timm_mnasnet::load_engine(context.backend, plan); - return new trtmc::TimmMnasnetImageClassificationPipeline(std::move(engine), std::move(config)); + return new trtmc::TimmMnasnetImageClassificationPipeline( + std::move(engine), std::move(config), metadata.at("num_classes").get(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_mnasnet/support.py b/families/timm_mnasnet/support.py index 541a9c8234..9f2ef54667 100644 --- a/families/timm_mnasnet/support.py +++ b/families/timm_mnasnet/support.py @@ -8,6 +8,6 @@ describe = family_support( architectures=("mnasnet_100",), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_mnasnet/tests/cpp/test_task_contract.cpp b/families/timm_mnasnet/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..8fecd371e8 --- /dev/null +++ b/families/timm_mnasnet/tests/cpp/test_task_contract.cpp @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_mnasnet/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{"logits", {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmMnasnetPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmMnasnetImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "", {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmMnasnetImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_mnasnet/tests/manifests/mnasnet-100-rmsp-in1k.json b/families/timm_mnasnet/tests/manifests/mnasnet-100-rmsp-in1k.json index 758cb24c16..2e4b369cdf 100644 --- a/families/timm_mnasnet/tests/manifests/mnasnet-100-rmsp-in1k.json +++ b/families/timm_mnasnet/tests/manifests/mnasnet-100-rmsp-in1k.json @@ -3,7 +3,7 @@ "hf_id": "timm/mnasnet_100.rmsp_in1k", "bundle": "mnasnet-100-rmsp-in1k.bundle", "family": "timm_mnasnet", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_mnasnet/tests/performance.yaml b/families/timm_mnasnet/tests/performance.yaml new file mode 100644 index 0000000000..7edee7f1d7 --- /dev/null +++ b/families/timm_mnasnet/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-mnasnet-performance +entries: + - id: timm_mnasnet.classify + family: timm_mnasnet + operation: classify + model: mnasnet-100-rmsp-in1k + workload: + testcase: mnasnet-100-rmsp-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_mnasnet/tests/performance_reference.py b/families/timm_mnasnet/tests/performance_reference.py new file mode 100644 index 0000000000..29b4bd7970 --- /dev/null +++ b/families/timm_mnasnet/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time MNASNet inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from safetensors.torch import load_file + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model(config["architecture"], pretrained=False, + num_classes=int(config["num_classes"])) + model.load_state_dict(load_file(str(checkpoint / "model.safetensors")), strict=True) + model.pretrained_cfg = config["pretrained_cfg"] + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("MNASNet must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("MNASNet returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_mnasnet" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_mnasnet image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_mnasnet/tests/sdk_consumer.c b/families/timm_mnasnet/tests/sdk_consumer.c new file mode 100644 index 0000000000..7a52476a9b --- /dev/null +++ b/families/timm_mnasnet/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "MNASNet must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "MNASNet must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_mnasnet/tests/sdk_consumer.cpp b/families/timm_mnasnet/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..7127956fe8 --- /dev/null +++ b/families/timm_mnasnet/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("MNASNet must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("MNASNet must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_mnasnet/tests/test_e2e.py b/families/timm_mnasnet/tests/test_e2e.py index 6f506eca8d..3e1e29a3b9 100644 --- a/families/timm_mnasnet/tests/test_e2e.py +++ b/families/timm_mnasnet/tests/test_e2e.py @@ -7,6 +7,7 @@ from tools.e2e_evidence import evidence_stage, record_evidence import json +import math import os import subprocess from pathlib import Path @@ -14,7 +15,7 @@ from tensorrt_model_connect import BuildRequest, build FAMILY = "timm_mnasnet" -TASKS = frozenset({"classification"}) +TASKS = frozenset({"image_to_class_scores"}) TEST_ROOT = Path(__file__).resolve().parent MANIFEST_ROOT = TEST_ROOT / "manifests" @@ -239,6 +240,58 @@ def _assert_parity(actual, expected) -> None: assert int(actual["top_class"]) == expected +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case["test_image"])).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_mnasnet_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: _, manifest, case = CASES[case_name] record_evidence("inputs", {"manifest": manifest, "case": CASES[case_name][-1]}) @@ -257,3 +310,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: record_evidence("reference", expected) with evidence_stage("compare"): _assert_parity(actual, expected) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert len(actual["scores"]) == int(config["num_classes"]) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, expected, tmp_path) diff --git a/families/timm_mnasnet/tests/test_model.py b/families/timm_mnasnet/tests/test_model.py index b17c734ff5..2f181a72ea 100644 --- a/families/timm_mnasnet/tests/test_model.py +++ b/families/timm_mnasnet/tests/test_model.py @@ -187,7 +187,7 @@ def test_build_rejects_unqualified_semnasnet_variant(tmp_path: Path): model_dir=tmp_path, output_path=tmp_path / "unused.bundle", family="timm_mnasnet", - task="classification", + task="image_to_class_scores", precision="fp16", max_sequence_length=1, ) @@ -201,10 +201,55 @@ def test_build_rejects_quantization(tmp_path: Path): model_dir=tmp_path, output_path=tmp_path / "unused.bundle", family="timm_mnasnet", - task="classification", + task="image_to_class_scores", precision="fp16", quantization="fp8", ) with pytest.raises(NotImplementedError, match="quantization"): build_family(request, object()) + + +@pytest.mark.parametrize("metadata", [ + {}, + {"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, +]) +def test_build_exports_task_and_checkpoint_metadata(tmp_path, monkeypatch, metadata): + _write_tiny_mnasnet(tmp_path) + raw = json.loads((tmp_path / "config.json").read_text()) + raw.update(metadata) + (tmp_path / "config.json").write_text(json.dumps(raw)) + monkeypatch.setattr(_TimmMnasnetModel, "build_engine", lambda *args, **kwargs: b"plan") + sections = {} + + class Writer: + def set_header(self, **value): + sections["header"] = value + + def add_bytes(self, name, value): + sections[name] = value + + def add_json(self, name, value): + sections[name] = value + + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_mnasnet", task="image_to_class_scores", precision="fp32") + build_family(request, Writer()) + assert sections["header"]["task"] == "image_to_class_scores" + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + + +@pytest.mark.parametrize("labels", [["only-one"], ["a", "b", "", "d", "e"], 5]) +def test_build_rejects_incomplete_class_labels(tmp_path, monkeypatch, labels): + _write_tiny_mnasnet(tmp_path) + raw = json.loads((tmp_path / "config.json").read_text()) + raw["label_names"] = labels + (tmp_path / "config.json").write_text(json.dumps(raw)) + monkeypatch.setattr(_TimmMnasnetModel, "build_engine", lambda *args, **kwargs: b"plan") + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_mnasnet", task="image_to_class_scores", precision="fp32") + with pytest.raises(ValueError, match="label_names must name every class"): + build_family(request, object()) diff --git a/families/timm_mnasnet/tests/test_performance_reference.py b/families/timm_mnasnet/tests/test_performance_reference.py new file mode 100644 index 0000000000..050513c140 --- /dev/null +++ b/families/timm_mnasnet/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_mnasnet.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_mnasnet", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_mnasnet.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_nfnet/README.md b/families/timm_nfnet/README.md new file mode 100644 index 0000000000..f3970434c5 --- /dev/null +++ b/families/timm_nfnet/README.md @@ -0,0 +1,66 @@ +# timm NFNet + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/dm_nfnet_f0.dm_in1k -o model.bundle +trtmc classify model.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_nfnet_task_contract test_timm_nfnet_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_nfnet_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_nfnet_sdk_c model.bundle build image.rgb.f32 480 640 +build/test_timm_nfnet_sdk_cpp model.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_nfnet.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_nfnet/model.py b/families/timm_nfnet/model.py index 630ef920ae..1903d3acaf 100644 --- a/families/timm_nfnet/model.py +++ b/families/timm_nfnet/model.py @@ -436,8 +436,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_nfnet does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_nfnet does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_nfnet supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_nfnet supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_nfnet does not support quantization") if request.fp32_layers: @@ -449,11 +449,23 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: plan, runtime = _build_engine( raw, Checkpoint.open(model_dir), str(request.precision).lower(), bool(request.verbose) ) + vocabulary_id = raw.get("vocabulary_id", "") + labels = raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm NFNet vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != runtime["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm NFNet label_names must name every class") writer.set_header(family="timm_nfnet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( "runtime.json", { + "num_classes": runtime["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "input_image_h": runtime["image_height"], "input_image_w": runtime["image_width"], "crop_pct": runtime["crop_pct"], diff --git a/families/timm_nfnet/runtime/CMakeLists.txt b/families/timm_nfnet/runtime/CMakeLists.txt index a7215f4ab9..3f11be3a6a 100644 --- a/families/timm_nfnet/runtime/CMakeLists.txt +++ b/families/timm_nfnet/runtime/CMakeLists.txt @@ -35,6 +35,41 @@ install(TARGETS trtmc_model_timm_nfnet ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_nfnet_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_nfnet/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_nfnet_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_nfnet_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_nfnet_task_contract PRIVATE + trtmc_model_timm_nfnet trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_nfnet_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_nfnet_task_contract COMMAND test_timm_nfnet_task_contract) + set_tests_properties(timm_nfnet_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_nfnet_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_nfnet/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_nfnet_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_nfnet_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_nfnet_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_nfnet_task_contract + test_timm_nfnet_sdk_c test_timm_nfnet_sdk_cpp + ) + add_executable(test_timm_nfnet_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_nfnet/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_nfnet/runtime/pipeline.cpp b/families/timm_nfnet/runtime/pipeline.cpp index 27814c96af..af0692d850 100644 --- a/families/timm_nfnet/runtime/pipeline.cpp +++ b/families/timm_nfnet/runtime/pipeline.cpp @@ -5,50 +5,78 @@ #include "families/timm_nfnet/runtime/pipeline.h" -#include #include +#include #include #include namespace trtmc { + namespace { -const Tensor* find_logits(const TensorMap& outputs) { +const Tensor& require_logits(const TensorMap& outputs) { for (const auto& [name, tensor] : outputs) { - if (name.find("logits") != std::string::npos || outputs.size() == 1) - return &tensor; + if (name.find("logits") == std::string::npos && outputs.size() != 1) + continue; + if (tensor.data == nullptr || tensor.dtype != DType::kFloat32 || tensor.numel() <= 0) + throw std::runtime_error("timm NFNet engine must return nonempty float32 logits"); + return tensor; } - return nullptr; + throw std::runtime_error("timm NFNet engine did not return logits"); } } // namespace TimmNfnetImageClassificationPipeline::TimmNfnetImageClassificationPipeline( - std::unique_ptr model, TimmNfnetPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmNfnetPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmNfnetImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm NFNet class metadata does not match its output size"); } -ClassificationResult TimmNfnetImageClassificationPipeline::classify(const float* pixels, - int32_t height, int32_t width) { - auto values = preprocess_timm_nfnet_image(pixels, height, width, preprocess_config_); - Tensor input; - input.data = values.data(); - input.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; - input.dtype = DType::kFloat32; - const auto outputs = model_->forward({{"pixel_values", input}}); - const Tensor* logits = find_logits(outputs); - if (logits == nullptr || logits->numel() <= 0) - throw std::runtime_error("timm NFNet engine returned no logits"); - if (logits->dtype != DType::kFloat32) - throw std::runtime_error("timm NFNet logits must be float32"); - ClassificationResult result; - result.logits.resize(static_cast(logits->numel())); - std::memcpy(result.logits.data(), logits->data, result.logits.size() * sizeof(float)); - const auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = *best; +internal::LabelScoresResult +TimmNfnetImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm NFNet has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm NFNet requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_nfnet_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); + + Tensor img_t; + img_t.data = pixel_values.data(); + img_t.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; + img_t.dtype = DType::kFloat32; + + auto outputs = model_->forward({{"pixel_values", img_t}}); + internal::LabelScoresResult result; + + const auto& logits_tensor = require_logits(outputs); + const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm NFNet logits do not match its configured class count"); + + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, + static_cast(n) * sizeof(float)); + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_nfnet/runtime/pipeline.h b/families/timm_nfnet/runtime/pipeline.h index 23ca4eede9..b6ac8fbc53 100644 --- a/families/timm_nfnet/runtime/pipeline.h +++ b/families/timm_nfnet/runtime/pipeline.h @@ -6,23 +6,36 @@ #pragma once #include "families/timm_nfnet/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmNfnetImageClassificationPipeline final : public IImageClassification { +class TimmNfnetImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: explicit TimmNfnetImageClassificationPipeline(std::unique_ptr model, - TimmNfnetPreprocessConfig preprocess_config = {}); - - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + TimmNfnetPreprocessConfig preprocess_config, + std::int32_t num_classes, + std::string vocabulary_id, + std::vector labels); + + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmNfnetPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_nfnet/runtime/plugin.cpp b/families/timm_nfnet/runtime/plugin.cpp index a496c9189e..d73a7bc84b 100644 --- a/families/timm_nfnet/runtime/plugin.cpp +++ b/families/timm_nfnet/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmNfnetPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmNfnetPreprocessConfig parse_config(const nlohmann::json& json) { TimmNfnetPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -36,7 +35,7 @@ TimmNfnetPreprocessConfig parse_config(const std::vector& data) { config.crop_pct > 1.0F || config.image_mean.size() != 3 || config.image_std.size() != 3 || (config.interpolation != "bilinear" && config.interpolation != "bicubic") || (config.crop_mode != "center" && config.crop_mode != "squash")) { - throw std::runtime_error("timm NFNet runtime.json does not match its contract"); + throw std::runtime_error("timm NFNet runtime.json does not match its runtime contract"); } return config; } @@ -55,9 +54,13 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_nfnet/support.py b/families/timm_nfnet/support.py index e12abdfb89..0f6408f0ec 100644 --- a/families/timm_nfnet/support.py +++ b/families/timm_nfnet/support.py @@ -26,6 +26,6 @@ "dm_nfnet_f5", "dm_nfnet_f6", ), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_nfnet/tests/cpp/test_task_contract.cpp b/families/timm_nfnet/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..03af896965 --- /dev/null +++ b/families/timm_nfnet/tests/cpp/test_task_contract.cpp @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_nfnet/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{output_name, {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + std::string output_name{"logits"}; + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmNfnetPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmNfnetImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmNfnetImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +void test_existing_named_output() { + auto module = std::make_unique(); + module->output_name = "class_scores"; + const auto expected = module->logits; + trtmc::TimmNfnetImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const std::vector pixels(12, 0.75F); + require(model.run(request(pixels), {}).scores == expected, + "a single named classifier output retains the existing runtime contract"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + test_existing_named_output(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_nfnet/tests/manifests/dm-nfnet-f0-dm-in1k.json b/families/timm_nfnet/tests/manifests/dm-nfnet-f0-dm-in1k.json index 3d17ce318b..4199b0840a 100644 --- a/families/timm_nfnet/tests/manifests/dm-nfnet-f0-dm-in1k.json +++ b/families/timm_nfnet/tests/manifests/dm-nfnet-f0-dm-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "ca8e830961c2ebceffff7affff34acfe7a1da673", "bundle": "dm-nfnet-f0-dm-in1k.bundle", "family": "timm_nfnet", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_nfnet/tests/manifests/dm-nfnet-f3-dm-in1k.json b/families/timm_nfnet/tests/manifests/dm-nfnet-f3-dm-in1k.json index b70d35e4d6..2ce21ca244 100644 --- a/families/timm_nfnet/tests/manifests/dm-nfnet-f3-dm-in1k.json +++ b/families/timm_nfnet/tests/manifests/dm-nfnet-f3-dm-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "ca2451bf0778ae0665a2f13d1b2a7fb878c9ff27", "bundle": "dm-nfnet-f3-dm-in1k.bundle", "family": "timm_nfnet", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_nfnet/tests/performance.yaml b/families/timm_nfnet/tests/performance.yaml new file mode 100644 index 0000000000..015cbdca19 --- /dev/null +++ b/families/timm_nfnet/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-nfnet-performance +entries: + - id: timm_nfnet.classify + family: timm_nfnet + operation: classify + model: dm-nfnet-f0-dm-in1k + workload: + testcase: dm-nfnet-f0-dm-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_nfnet/tests/performance_reference.py b/families/timm_nfnet/tests/performance_reference.py new file mode 100644 index 0000000000..50eb378a1c --- /dev/null +++ b/families/timm_nfnet/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time NFNet inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model( + config["architecture"], pretrained=False, + pretrained_cfg=config["pretrained_cfg"], num_classes=int(config["num_classes"]), + checkpoint_path=str(checkpoint / "model.safetensors"), + ) + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("NFNet must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("NFNet returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_nfnet" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_nfnet image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_nfnet/tests/sdk_consumer.c b/families/timm_nfnet/tests/sdk_consumer.c new file mode 100644 index 0000000000..d5ec959f28 --- /dev/null +++ b/families/timm_nfnet/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "NFNet must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "NFNet must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_nfnet/tests/sdk_consumer.cpp b/families/timm_nfnet/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..d387fafd0f --- /dev/null +++ b/families/timm_nfnet/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("NFNet must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("NFNet must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_nfnet/tests/test_e2e.py b/families/timm_nfnet/tests/test_e2e.py index 134a6193f7..31615c4d55 100644 --- a/families/timm_nfnet/tests/test_e2e.py +++ b/families/timm_nfnet/tests/test_e2e.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import math import os import subprocess from pathlib import Path @@ -27,7 +28,7 @@ def _cases() -> dict[str, tuple[dict, dict]]: for path in sorted(MANIFEST_ROOT.glob("*.json")): manifest = json.loads(path.read_text(encoding="utf-8")) assert manifest["family"] == FAMILY - assert manifest["task"] == "classification" + assert manifest["task"] == "image_to_class_scores" for case in manifest["testcases"]: name = str(case["name"]) assert name not in result @@ -104,6 +105,58 @@ def _asset(case: dict) -> Path: return path +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case)).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_nfnet_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: manifest, case = CASES[case_name] record_evidence( @@ -186,3 +239,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: record_evidence("reference", {"top_class": int(np.argmax(expected)), "logits": expected}) with evidence_stage("compare"): assert int(actual["top_class"]) == int(np.argmax(expected)) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert len(actual["scores"]) == int(config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, int(np.argmax(expected)), tmp_path) diff --git a/families/timm_nfnet/tests/test_performance_reference.py b/families/timm_nfnet/tests/test_performance_reference.py new file mode 100644 index 0000000000..04b1f68f79 --- /dev/null +++ b/families/timm_nfnet/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_nfnet.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_nfnet", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_nfnet.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_nfnet/tests/test_sdk_metadata.py b/families/timm_nfnet/tests/test_sdk_metadata.py new file mode 100644 index 0000000000..99b39ddf01 --- /dev/null +++ b/families/timm_nfnet/tests/test_sdk_metadata.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The builder publishes complete checkpoint-owned Task metadata.""" + +from types import SimpleNamespace + +import pytest + +from families.timm_nfnet import model +from tensorrt_model_connect import BuildRequest + + +@pytest.mark.parametrize("metadata,invalid", [ + ({}, False), + ({"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, False), + ({"label_names": ["only-one"]}, True), + ({"label_names": ["a", "b", "", "d", "e"]}, True), + ({"label_names": 5}, True), + ({"vocabulary_id": 5}, True), +]) +def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): + raw = {"num_classes": 5, **metadata} + runtime = {"image_height": 2, "image_width": 2, "num_classes": 5, + "crop_pct": 1.0, "interpolation": "bilinear", "crop_mode": "squash", + "mean": [0.5] * 3, "std": [0.25] * 3} + monkeypatch.setattr(model, "_read_config", lambda _: raw) + monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) + monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + sections = {} + headers = [] + writer = SimpleNamespace(set_header=lambda **value: headers.append(value), + add_bytes=lambda key, value: sections.update({key: value}), + add_json=lambda key, value: sections.update({key: value})) + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_nfnet", task="image_to_class_scores", precision="fp32") + if invalid: + with pytest.raises(ValueError, match="vocabulary_id|label_names"): + model.build(request, writer) + assert not sections and not headers + return + model.build(request, writer) + assert headers == [{"family": "timm_nfnet", "task": "image_to_class_scores", "backend": "trt"}] + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) + assert sections["runtime.json"]["crop_mode"] == "squash" diff --git a/families/timm_repvgg/README.md b/families/timm_repvgg/README.md new file mode 100644 index 0000000000..62d44d5497 --- /dev/null +++ b/families/timm_repvgg/README.md @@ -0,0 +1,66 @@ +# timm RepVGG + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/repvgg_a2.rvgg_in1k -o model.bundle +trtmc classify model.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_repvgg_task_contract test_timm_repvgg_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_repvgg_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_repvgg_sdk_c model.bundle build image.rgb.f32 480 640 +build/test_timm_repvgg_sdk_cpp model.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_repvgg.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_repvgg/model.py b/families/timm_repvgg/model.py index 1d4bde114d..29868c10c2 100644 --- a/families/timm_repvgg/model.py +++ b/families/timm_repvgg/model.py @@ -329,8 +329,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_repvgg does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_repvgg does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_repvgg supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_repvgg supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_repvgg does not support quantization") if request.fp32_layers: @@ -344,11 +344,23 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: str(request.precision).lower(), bool(request.verbose), ) + vocabulary_id = raw.get("vocabulary_id", "") + labels = raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm RepVGG vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != runtime["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm RepVGG label_names must name every class") writer.set_header(family="timm_repvgg", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( "runtime.json", { + "num_classes": runtime["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "input_image_h": runtime["image_height"], "input_image_w": runtime["image_width"], "crop_pct": runtime["crop_pct"], diff --git a/families/timm_repvgg/runtime/CMakeLists.txt b/families/timm_repvgg/runtime/CMakeLists.txt index dc47600c2b..ee83530cdc 100644 --- a/families/timm_repvgg/runtime/CMakeLists.txt +++ b/families/timm_repvgg/runtime/CMakeLists.txt @@ -35,6 +35,41 @@ install(TARGETS trtmc_model_timm_repvgg ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_repvgg_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_repvgg/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_repvgg_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_repvgg_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_repvgg_task_contract PRIVATE + trtmc_model_timm_repvgg trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_repvgg_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_repvgg_task_contract COMMAND test_timm_repvgg_task_contract) + set_tests_properties(timm_repvgg_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_repvgg_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_repvgg/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_repvgg_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_repvgg_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_repvgg_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_repvgg_task_contract + test_timm_repvgg_sdk_c test_timm_repvgg_sdk_cpp + ) + add_executable(test_timm_repvgg_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_repvgg/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_repvgg/runtime/pipeline.cpp b/families/timm_repvgg/runtime/pipeline.cpp index fbb7ea0e65..002756ccfb 100644 --- a/families/timm_repvgg/runtime/pipeline.cpp +++ b/families/timm_repvgg/runtime/pipeline.cpp @@ -5,51 +5,78 @@ #include "families/timm_repvgg/runtime/pipeline.h" -#include #include +#include #include #include namespace trtmc { + namespace { -const Tensor* find_logits(const TensorMap& outputs) { +const Tensor& require_logits(const TensorMap& outputs) { for (const auto& [name, tensor] : outputs) { - if (name.find("logits") != std::string::npos || outputs.size() == 1) - return &tensor; + if (name.find("logits") == std::string::npos && outputs.size() != 1) + continue; + if (tensor.data == nullptr || tensor.dtype != DType::kFloat32 || tensor.numel() <= 0) + throw std::runtime_error("timm RepVGG engine must return nonempty float32 logits"); + return tensor; } - return nullptr; + throw std::runtime_error("timm RepVGG engine did not return logits"); } } // namespace TimmRepvggImageClassificationPipeline::TimmRepvggImageClassificationPipeline( - std::unique_ptr model, TimmRepvggPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmRepvggPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmRepvggImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm RepVGG class metadata does not match its output size"); } -ClassificationResult TimmRepvggImageClassificationPipeline::classify(const float* pixels, - int32_t height, - int32_t width) { - auto values = preprocess_timm_repvgg_image(pixels, height, width, preprocess_config_); - Tensor input; - input.data = values.data(); - input.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; - input.dtype = DType::kFloat32; - const auto outputs = model_->forward({{"pixel_values", input}}); - const Tensor* logits = find_logits(outputs); - if (logits == nullptr || logits->numel() <= 0) - throw std::runtime_error("timm RepVGG engine returned no logits"); - if (logits->dtype != DType::kFloat32) - throw std::runtime_error("timm RepVGG logits must be float32"); - ClassificationResult result; - result.logits.resize(static_cast(logits->numel())); - std::memcpy(result.logits.data(), logits->data, result.logits.size() * sizeof(float)); - const auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = *best; +internal::LabelScoresResult +TimmRepvggImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm RepVGG has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm RepVGG requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_repvgg_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); + + Tensor img_t; + img_t.data = pixel_values.data(); + img_t.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; + img_t.dtype = DType::kFloat32; + + auto outputs = model_->forward({{"pixel_values", img_t}}); + internal::LabelScoresResult result; + + const auto& logits_tensor = require_logits(outputs); + const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm RepVGG logits do not match its configured class count"); + + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, + static_cast(n) * sizeof(float)); + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_repvgg/runtime/pipeline.h b/families/timm_repvgg/runtime/pipeline.h index c6c32dcae7..27a546c531 100644 --- a/families/timm_repvgg/runtime/pipeline.h +++ b/families/timm_repvgg/runtime/pipeline.h @@ -6,23 +6,36 @@ #pragma once #include "families/timm_repvgg/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmRepvggImageClassificationPipeline final : public IImageClassification { +class TimmRepvggImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: - explicit TimmRepvggImageClassificationPipeline( - std::unique_ptr model, TimmRepvggPreprocessConfig preprocess_config = {}); - - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + explicit TimmRepvggImageClassificationPipeline(std::unique_ptr model, + TimmRepvggPreprocessConfig preprocess_config, + std::int32_t num_classes, + std::string vocabulary_id, + std::vector labels); + + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmRepvggPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_repvgg/runtime/plugin.cpp b/families/timm_repvgg/runtime/plugin.cpp index d03555aa18..4bca0c261f 100644 --- a/families/timm_repvgg/runtime/plugin.cpp +++ b/families/timm_repvgg/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmRepvggPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmRepvggPreprocessConfig parse_config(const nlohmann::json& json) { TimmRepvggPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -34,7 +33,7 @@ TimmRepvggPreprocessConfig parse_config(const std::vector& data) { if (config.input_image_h <= 0 || config.input_image_w <= 0 || config.crop_pct <= 0.0F || config.crop_pct > 1.0F || config.image_mean.size() != 3 || config.image_std.size() != 3 || (config.interpolation != "bilinear" && config.interpolation != "bicubic")) { - throw std::runtime_error("timm RepVGG runtime.json does not match its contract"); + throw std::runtime_error("timm RepVGG runtime.json does not match its runtime contract"); } return config; } @@ -53,9 +52,13 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_repvgg/support.py b/families/timm_repvgg/support.py index f83359ac42..e543d51744 100644 --- a/families/timm_repvgg/support.py +++ b/families/timm_repvgg/support.py @@ -9,6 +9,6 @@ describe = family_support( model_types=("timm_repvgg", "repvgg_a2"), architectures=("repvgg_a2",), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_repvgg/tests/cpp/test_task_contract.cpp b/families/timm_repvgg/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..410cf566a7 --- /dev/null +++ b/families/timm_repvgg/tests/cpp/test_task_contract.cpp @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_repvgg/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{output_name, {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + std::string output_name{"logits"}; + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmRepvggPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmRepvggImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "", {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmRepvggImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +void test_existing_named_output() { + auto module = std::make_unique(); + module->output_name = "class_scores"; + const auto expected = module->logits; + trtmc::TimmRepvggImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const std::vector pixels(12, 0.75F); + require(model.run(request(pixels), {}).scores == expected, + "a single named classifier output retains the existing runtime contract"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + test_existing_named_output(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_repvgg/tests/manifests/repvgg-a2-rvgg-in1k.json b/families/timm_repvgg/tests/manifests/repvgg-a2-rvgg-in1k.json index 59dc8ce844..89c01fccef 100644 --- a/families/timm_repvgg/tests/manifests/repvgg-a2-rvgg-in1k.json +++ b/families/timm_repvgg/tests/manifests/repvgg-a2-rvgg-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "87d4d383cb45031cb9fa2fc8ddca73fd6649240f", "bundle": "repvgg-a2-rvgg-in1k.bundle", "family": "timm_repvgg", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_repvgg/tests/performance.yaml b/families/timm_repvgg/tests/performance.yaml new file mode 100644 index 0000000000..7e42cb1d97 --- /dev/null +++ b/families/timm_repvgg/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-repvgg-performance +entries: + - id: timm_repvgg.classify + family: timm_repvgg + operation: classify + model: repvgg-a2-rvgg-in1k + workload: + testcase: repvgg-a2-rvgg-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_repvgg/tests/performance_reference.py b/families/timm_repvgg/tests/performance_reference.py new file mode 100644 index 0000000000..30728d9283 --- /dev/null +++ b/families/timm_repvgg/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time RepVGG inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model( + config["architecture"], pretrained=False, + pretrained_cfg=config["pretrained_cfg"], num_classes=int(config["num_classes"]), + checkpoint_path=str(checkpoint / "model.safetensors"), + ) + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("RepVGG must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("RepVGG returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_repvgg" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_repvgg image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_repvgg/tests/sdk_consumer.c b/families/timm_repvgg/tests/sdk_consumer.c new file mode 100644 index 0000000000..164501125b --- /dev/null +++ b/families/timm_repvgg/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "RepVGG must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "RepVGG must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_repvgg/tests/sdk_consumer.cpp b/families/timm_repvgg/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..f0be50f04f --- /dev/null +++ b/families/timm_repvgg/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("RepVGG must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("RepVGG must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_repvgg/tests/test_e2e.py b/families/timm_repvgg/tests/test_e2e.py index 93c9296d7f..8c9a82114b 100644 --- a/families/timm_repvgg/tests/test_e2e.py +++ b/families/timm_repvgg/tests/test_e2e.py @@ -8,6 +8,7 @@ from tools.e2e_evidence import evidence_stage, record_evidence import json +import math import os import subprocess from pathlib import Path @@ -28,7 +29,7 @@ def _cases() -> dict[str, tuple[dict, dict]]: for path in sorted(MANIFEST_ROOT.glob("*.json")): manifest = json.loads(path.read_text(encoding="utf-8")) assert manifest["family"] == FAMILY - assert manifest["task"] == "classification" + assert manifest["task"] == "image_to_class_scores" for case in manifest["testcases"]: name = str(case["name"]) assert name not in result @@ -105,6 +106,58 @@ def _asset(case: dict) -> Path: return path +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case)).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_repvgg_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: manifest, case = CASES[case_name] record_evidence("inputs", {"manifest": manifest, "case": CASES[case_name][-1]}) @@ -177,3 +230,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: ) with evidence_stage("compare"): assert int(actual["top_class"]) == int(np.argmax(expected)) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert len(actual["scores"]) == int(config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, int(np.argmax(expected)), tmp_path) diff --git a/families/timm_repvgg/tests/test_model.py b/families/timm_repvgg/tests/test_model.py index df0e8cb75c..ef103bff2d 100644 --- a/families/timm_repvgg/tests/test_model.py +++ b/families/timm_repvgg/tests/test_model.py @@ -167,7 +167,7 @@ def add_json(self, name, value) -> None: backend="trt", dynamic_kv_cache=False, family="timm_repvgg", - task="classification", + task="image_to_class_scores", precision="fp16", max_sequence_length=1, image_height=None, @@ -186,7 +186,7 @@ def add_json(self, name, value) -> None: assert writer.header == { "family": "timm_repvgg", - "task": "classification", + "task": "image_to_class_scores", "backend": "trt", } assert writer.sections["engine.plan"] == b"plan" diff --git a/families/timm_repvgg/tests/test_performance_reference.py b/families/timm_repvgg/tests/test_performance_reference.py new file mode 100644 index 0000000000..5e3b143991 --- /dev/null +++ b/families/timm_repvgg/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_repvgg.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_repvgg", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_repvgg.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_repvgg/tests/test_sdk_metadata.py b/families/timm_repvgg/tests/test_sdk_metadata.py new file mode 100644 index 0000000000..88ed9dd61b --- /dev/null +++ b/families/timm_repvgg/tests/test_sdk_metadata.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The builder publishes complete checkpoint-owned Task metadata.""" + +from types import SimpleNamespace + +import pytest + +from families.timm_repvgg import model +from tensorrt_model_connect import BuildRequest + + +@pytest.mark.parametrize("metadata,invalid", [ + ({}, False), + ({"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, False), + ({"label_names": ["only-one"]}, True), + ({"label_names": ["a", "b", "", "d", "e"]}, True), + ({"label_names": 5}, True), + ({"vocabulary_id": 5}, True), +]) +def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): + raw = {"num_classes": 5, **metadata} + runtime = {"image_height": 2, "image_width": 2, "num_classes": 5, + "crop_pct": 1.0, "interpolation": "bilinear", + "mean": [0.5] * 3, "std": [0.25] * 3} + monkeypatch.setattr(model, "_read_config", lambda _: raw) + monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) + monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + sections = {} + headers = [] + writer = SimpleNamespace(set_header=lambda **value: headers.append(value), + add_bytes=lambda key, value: sections.update({key: value}), + add_json=lambda key, value: sections.update({key: value})) + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_repvgg", task="image_to_class_scores", precision="fp32") + if invalid: + with pytest.raises(ValueError, match="vocabulary_id|label_names"): + model.build(request, writer) + assert not sections and not headers + return + model.build(request, writer) + assert headers == [{"family": "timm_repvgg", "task": "image_to_class_scores", "backend": "trt"}] + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) diff --git a/families/timm_res2net/README.md b/families/timm_res2net/README.md new file mode 100644 index 0000000000..1c663ce388 --- /dev/null +++ b/families/timm_res2net/README.md @@ -0,0 +1,66 @@ +# timm Res2Net + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/res2net50_26w_4s.in1k -o model.bundle +trtmc classify model.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_res2net_task_contract test_timm_res2net_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_res2net_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_res2net_sdk_c model.bundle build image.rgb.f32 480 640 +build/test_timm_res2net_sdk_cpp model.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_res2net.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_res2net/model.py b/families/timm_res2net/model.py index cd10c3c66c..73418fb114 100644 --- a/families/timm_res2net/model.py +++ b/families/timm_res2net/model.py @@ -470,8 +470,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_res2net does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_res2net does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_res2net supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_res2net supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_res2net does not support quantization") if request.fp32_layers: @@ -485,11 +485,23 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: str(request.precision).lower(), bool(request.verbose), ) + vocabulary_id = raw.get("vocabulary_id", "") + labels = raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm Res2Net vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != runtime["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm Res2Net label_names must name every class") writer.set_header(family="timm_res2net", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( "runtime.json", { + "num_classes": runtime["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "input_image_h": runtime["image_height"], "input_image_w": runtime["image_width"], "crop_pct": runtime["crop_pct"], diff --git a/families/timm_res2net/runtime/CMakeLists.txt b/families/timm_res2net/runtime/CMakeLists.txt index a4553f53dc..3f1e2b2789 100644 --- a/families/timm_res2net/runtime/CMakeLists.txt +++ b/families/timm_res2net/runtime/CMakeLists.txt @@ -35,6 +35,41 @@ install(TARGETS trtmc_model_timm_res2net ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_res2net_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_res2net/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_res2net_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_res2net_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_res2net_task_contract PRIVATE + trtmc_model_timm_res2net trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_res2net_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_res2net_task_contract COMMAND test_timm_res2net_task_contract) + set_tests_properties(timm_res2net_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_res2net_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_res2net/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_res2net_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_res2net_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_res2net_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_res2net_task_contract + test_timm_res2net_sdk_c test_timm_res2net_sdk_cpp + ) + add_executable(test_timm_res2net_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_res2net/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_res2net/runtime/pipeline.cpp b/families/timm_res2net/runtime/pipeline.cpp index 859e85ceba..1a74bf35fc 100644 --- a/families/timm_res2net/runtime/pipeline.cpp +++ b/families/timm_res2net/runtime/pipeline.cpp @@ -5,51 +5,78 @@ #include "families/timm_res2net/runtime/pipeline.h" -#include #include +#include #include #include namespace trtmc { + namespace { -const Tensor* find_logits(const TensorMap& outputs) { +const Tensor& require_logits(const TensorMap& outputs) { for (const auto& [name, tensor] : outputs) { - if (name.find("logits") != std::string::npos || outputs.size() == 1) - return &tensor; + if (name.find("logits") == std::string::npos && outputs.size() != 1) + continue; + if (tensor.data == nullptr || tensor.dtype != DType::kFloat32 || tensor.numel() <= 0) + throw std::runtime_error("timm Res2Net engine must return nonempty float32 logits"); + return tensor; } - return nullptr; + throw std::runtime_error("timm Res2Net engine did not return logits"); } } // namespace TimmRes2NetImageClassificationPipeline::TimmRes2NetImageClassificationPipeline( - std::unique_ptr model, TimmRes2NetPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmRes2NetPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmRes2NetImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm Res2Net class metadata does not match its output size"); } -ClassificationResult TimmRes2NetImageClassificationPipeline::classify(const float* pixels, - int32_t height, - int32_t width) { - auto values = preprocess_timm_res2net_image(pixels, height, width, preprocess_config_); - Tensor input; - input.data = values.data(); - input.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; - input.dtype = DType::kFloat32; - const auto outputs = model_->forward({{"pixel_values", input}}); - const Tensor* logits = find_logits(outputs); - if (logits == nullptr || logits->numel() <= 0) - throw std::runtime_error("timm Res2Net engine returned no logits"); - if (logits->dtype != DType::kFloat32) - throw std::runtime_error("timm Res2Net logits must be float32"); - ClassificationResult result; - result.logits.resize(static_cast(logits->numel())); - std::memcpy(result.logits.data(), logits->data, result.logits.size() * sizeof(float)); - const auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = *best; +internal::LabelScoresResult +TimmRes2NetImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm Res2Net has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm Res2Net requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_res2net_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); + + Tensor img_t; + img_t.data = pixel_values.data(); + img_t.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; + img_t.dtype = DType::kFloat32; + + auto outputs = model_->forward({{"pixel_values", img_t}}); + internal::LabelScoresResult result; + + const auto& logits_tensor = require_logits(outputs); + const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm Res2Net logits do not match its configured class count"); + + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, + static_cast(n) * sizeof(float)); + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_res2net/runtime/pipeline.h b/families/timm_res2net/runtime/pipeline.h index 0ee209c739..0f5fd6bed4 100644 --- a/families/timm_res2net/runtime/pipeline.h +++ b/families/timm_res2net/runtime/pipeline.h @@ -6,23 +6,36 @@ #pragma once #include "families/timm_res2net/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmRes2NetImageClassificationPipeline final : public IImageClassification { +class TimmRes2NetImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: - explicit TimmRes2NetImageClassificationPipeline( - std::unique_ptr model, TimmRes2NetPreprocessConfig preprocess_config = {}); - - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + explicit TimmRes2NetImageClassificationPipeline(std::unique_ptr model, + TimmRes2NetPreprocessConfig preprocess_config, + std::int32_t num_classes, + std::string vocabulary_id, + std::vector labels); + + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmRes2NetPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_res2net/runtime/plugin.cpp b/families/timm_res2net/runtime/plugin.cpp index 16d8b71ff0..156f15e784 100644 --- a/families/timm_res2net/runtime/plugin.cpp +++ b/families/timm_res2net/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmRes2NetPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmRes2NetPreprocessConfig parse_config(const nlohmann::json& json) { TimmRes2NetPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -34,7 +33,7 @@ TimmRes2NetPreprocessConfig parse_config(const std::vector& data) { if (config.input_image_h <= 0 || config.input_image_w <= 0 || config.crop_pct <= 0.0F || config.crop_pct > 1.0F || config.image_mean.size() != 3 || config.image_std.size() != 3 || (config.interpolation != "bilinear" && config.interpolation != "bicubic")) { - throw std::runtime_error("timm Res2Net runtime.json does not match its contract"); + throw std::runtime_error("timm Res2Net runtime.json does not match its runtime contract"); } return config; } @@ -53,9 +52,13 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_res2net/support.py b/families/timm_res2net/support.py index 7431c5961a..35549c06e2 100644 --- a/families/timm_res2net/support.py +++ b/families/timm_res2net/support.py @@ -30,6 +30,6 @@ "res2net101d", "res2next50", ), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_res2net/tests/cpp/test_task_contract.cpp b/families/timm_res2net/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..b7bfdbb646 --- /dev/null +++ b/families/timm_res2net/tests/cpp/test_task_contract.cpp @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_res2net/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{output_name, {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + std::string output_name{"logits"}; + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmRes2NetPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmRes2NetImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "", {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmRes2NetImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +void test_existing_named_output() { + auto module = std::make_unique(); + module->output_name = "class_scores"; + const auto expected = module->logits; + trtmc::TimmRes2NetImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const std::vector pixels(12, 0.75F); + require(model.run(request(pixels), {}).scores == expected, + "a single named classifier output retains the existing runtime contract"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + test_existing_named_output(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_res2net/tests/manifests/res2net50-26w-4s-in1k.json b/families/timm_res2net/tests/manifests/res2net50-26w-4s-in1k.json index 6c845bcf0e..86677c444b 100644 --- a/families/timm_res2net/tests/manifests/res2net50-26w-4s-in1k.json +++ b/families/timm_res2net/tests/manifests/res2net50-26w-4s-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "a36466d0354c9d337dbc0af329e13cb1cd7137fe", "bundle": "res2net50-26w-4s-in1k.bundle", "family": "timm_res2net", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_res2net/tests/manifests/res2net50d-in1k.json b/families/timm_res2net/tests/manifests/res2net50d-in1k.json index 0fafb42530..98d3c7301d 100644 --- a/families/timm_res2net/tests/manifests/res2net50d-in1k.json +++ b/families/timm_res2net/tests/manifests/res2net50d-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "4f4bce4a1b5c17eabe5dde59af578ff9e1199ffc", "bundle": "res2net50d-in1k.bundle", "family": "timm_res2net", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_res2net/tests/manifests/res2next50-in1k.json b/families/timm_res2net/tests/manifests/res2next50-in1k.json index c71a160f7b..a71fc1711b 100644 --- a/families/timm_res2net/tests/manifests/res2next50-in1k.json +++ b/families/timm_res2net/tests/manifests/res2next50-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "a8fe0f89e842b6dfa74b96deed2254d4fb7bf625", "bundle": "res2next50-in1k.bundle", "family": "timm_res2net", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_res2net/tests/performance.yaml b/families/timm_res2net/tests/performance.yaml new file mode 100644 index 0000000000..24c4aa7b3a --- /dev/null +++ b/families/timm_res2net/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-res2net-performance +entries: + - id: timm_res2net.classify + family: timm_res2net + operation: classify + model: res2net50-26w-4s-in1k + workload: + testcase: res2net50-26w-4s-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_res2net/tests/performance_reference.py b/families/timm_res2net/tests/performance_reference.py new file mode 100644 index 0000000000..67001934b9 --- /dev/null +++ b/families/timm_res2net/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time Res2Net inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model( + config["architecture"], pretrained=False, + pretrained_cfg=config["pretrained_cfg"], num_classes=int(config["num_classes"]), + checkpoint_path=str(checkpoint / "model.safetensors"), + ) + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("Res2Net must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("Res2Net returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_res2net" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_res2net image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_res2net/tests/sdk_consumer.c b/families/timm_res2net/tests/sdk_consumer.c new file mode 100644 index 0000000000..962d24994d --- /dev/null +++ b/families/timm_res2net/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "Res2Net must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "Res2Net must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_res2net/tests/sdk_consumer.cpp b/families/timm_res2net/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..d213cd634c --- /dev/null +++ b/families/timm_res2net/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("Res2Net must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("Res2Net must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_res2net/tests/test_e2e.py b/families/timm_res2net/tests/test_e2e.py index ab2b4f975e..f092dcf2c8 100644 --- a/families/timm_res2net/tests/test_e2e.py +++ b/families/timm_res2net/tests/test_e2e.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import math import os import subprocess from pathlib import Path @@ -27,7 +28,7 @@ def _cases() -> dict[str, tuple[dict, dict]]: for path in sorted(MANIFEST_ROOT.glob("*.json")): manifest = json.loads(path.read_text(encoding="utf-8")) assert manifest["family"] == FAMILY - assert manifest["task"] == "classification" + assert manifest["task"] == "image_to_class_scores" for case in manifest["testcases"]: name = str(case["name"]) assert name not in result @@ -104,6 +105,58 @@ def _asset(case: dict) -> Path: return path +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case)).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_res2net_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: manifest, case = CASES[case_name] record_evidence("inputs", {"manifest": manifest, "case": case}) @@ -176,3 +229,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: record_evidence("reference", {"top_class": int(np.argmax(expected)), "logits": expected}) with evidence_stage("compare"): assert int(actual["top_class"]) == int(np.argmax(expected)) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert len(actual["scores"]) == int(config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, int(np.argmax(expected)), tmp_path) diff --git a/families/timm_res2net/tests/test_performance_reference.py b/families/timm_res2net/tests/test_performance_reference.py new file mode 100644 index 0000000000..35d3166027 --- /dev/null +++ b/families/timm_res2net/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_res2net.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_res2net", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_res2net.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_res2net/tests/test_sdk_metadata.py b/families/timm_res2net/tests/test_sdk_metadata.py new file mode 100644 index 0000000000..87748571fe --- /dev/null +++ b/families/timm_res2net/tests/test_sdk_metadata.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The builder publishes complete checkpoint-owned Task metadata.""" + +from types import SimpleNamespace + +import pytest + +from families.timm_res2net import model +from tensorrt_model_connect import BuildRequest + + +@pytest.mark.parametrize("metadata,invalid", [ + ({}, False), + ({"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, False), + ({"label_names": ["only-one"]}, True), + ({"label_names": ["a", "b", "", "d", "e"]}, True), + ({"label_names": 5}, True), + ({"vocabulary_id": 5}, True), +]) +def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): + raw = {"num_classes": 5, **metadata} + runtime = {"image_height": 2, "image_width": 2, "num_classes": 5, + "crop_pct": 1.0, "interpolation": "bilinear", + "mean": [0.5] * 3, "std": [0.25] * 3} + monkeypatch.setattr(model, "_read_config", lambda _: raw) + monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) + monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + sections = {} + headers = [] + writer = SimpleNamespace(set_header=lambda **value: headers.append(value), + add_bytes=lambda key, value: sections.update({key: value}), + add_json=lambda key, value: sections.update({key: value})) + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_res2net", task="image_to_class_scores", precision="fp32") + if invalid: + with pytest.raises(ValueError, match="vocabulary_id|label_names"): + model.build(request, writer) + assert not sections and not headers + return + model.build(request, writer) + assert headers == [{"family": "timm_res2net", "task": "image_to_class_scores", "backend": "trt"}] + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) diff --git a/families/timm_senet/README.md b/families/timm_senet/README.md new file mode 100644 index 0000000000..bf5e35ed8c --- /dev/null +++ b/families/timm_senet/README.md @@ -0,0 +1,66 @@ +# timm SENet + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/senet154.gluon_in1k -o model.bundle +trtmc classify model.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_senet_task_contract test_timm_senet_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_senet_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_senet_sdk_c model.bundle build image.rgb.f32 480 640 +build/test_timm_senet_sdk_cpp model.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_senet.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_senet/model.py b/families/timm_senet/model.py index 54cef0bac6..ed0274e12c 100644 --- a/families/timm_senet/model.py +++ b/families/timm_senet/model.py @@ -355,8 +355,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_senet does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_senet does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_senet supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_senet supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_senet does not support quantization") if request.fp32_layers: @@ -370,11 +370,23 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: str(request.precision).lower(), bool(request.verbose), ) + vocabulary_id = raw.get("vocabulary_id", "") + labels = raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm SENet vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != runtime["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm SENet label_names must name every class") writer.set_header(family="timm_senet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( "runtime.json", { + "num_classes": runtime["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "input_image_h": runtime["image_height"], "input_image_w": runtime["image_width"], "crop_pct": runtime["crop_pct"], diff --git a/families/timm_senet/runtime/CMakeLists.txt b/families/timm_senet/runtime/CMakeLists.txt index e8ce9a840b..94e96995a4 100644 --- a/families/timm_senet/runtime/CMakeLists.txt +++ b/families/timm_senet/runtime/CMakeLists.txt @@ -35,6 +35,41 @@ install(TARGETS trtmc_model_timm_senet ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_senet_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_senet/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_senet_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_senet_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_senet_task_contract PRIVATE + trtmc_model_timm_senet trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_senet_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_senet_task_contract COMMAND test_timm_senet_task_contract) + set_tests_properties(timm_senet_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_senet_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_senet/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_senet_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_senet_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_senet_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_senet_task_contract + test_timm_senet_sdk_c test_timm_senet_sdk_cpp + ) + add_executable(test_timm_senet_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_senet/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_senet/runtime/pipeline.cpp b/families/timm_senet/runtime/pipeline.cpp index e84ca99151..12524781fe 100644 --- a/families/timm_senet/runtime/pipeline.cpp +++ b/families/timm_senet/runtime/pipeline.cpp @@ -5,50 +5,78 @@ #include "families/timm_senet/runtime/pipeline.h" -#include #include +#include #include #include namespace trtmc { + namespace { -const Tensor* find_logits(const TensorMap& outputs) { +const Tensor& require_logits(const TensorMap& outputs) { for (const auto& [name, tensor] : outputs) { - if (name.find("logits") != std::string::npos || outputs.size() == 1) - return &tensor; + if (name.find("logits") == std::string::npos && outputs.size() != 1) + continue; + if (tensor.data == nullptr || tensor.dtype != DType::kFloat32 || tensor.numel() <= 0) + throw std::runtime_error("timm SENet engine must return nonempty float32 logits"); + return tensor; } - return nullptr; + throw std::runtime_error("timm SENet engine did not return logits"); } } // namespace TimmSENetImageClassificationPipeline::TimmSENetImageClassificationPipeline( - std::unique_ptr model, TimmSENetPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmSENetPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmSENetImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm SENet class metadata does not match its output size"); } -ClassificationResult TimmSENetImageClassificationPipeline::classify(const float* pixels, - int32_t height, int32_t width) { - auto values = preprocess_timm_senet_image(pixels, height, width, preprocess_config_); - Tensor input; - input.data = values.data(); - input.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; - input.dtype = DType::kFloat32; - const auto outputs = model_->forward({{"pixel_values", input}}); - const Tensor* logits = find_logits(outputs); - if (logits == nullptr || logits->numel() <= 0) - throw std::runtime_error("timm SENet engine returned no logits"); - if (logits->dtype != DType::kFloat32) - throw std::runtime_error("timm SENet logits must be float32"); - ClassificationResult result; - result.logits.resize(static_cast(logits->numel())); - std::memcpy(result.logits.data(), logits->data, result.logits.size() * sizeof(float)); - const auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = *best; +internal::LabelScoresResult +TimmSENetImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm SENet has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm SENet requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_senet_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); + + Tensor img_t; + img_t.data = pixel_values.data(); + img_t.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; + img_t.dtype = DType::kFloat32; + + auto outputs = model_->forward({{"pixel_values", img_t}}); + internal::LabelScoresResult result; + + const auto& logits_tensor = require_logits(outputs); + const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm SENet logits do not match its configured class count"); + + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, + static_cast(n) * sizeof(float)); + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_senet/runtime/pipeline.h b/families/timm_senet/runtime/pipeline.h index 5af8dfc60a..d37e34e6c8 100644 --- a/families/timm_senet/runtime/pipeline.h +++ b/families/timm_senet/runtime/pipeline.h @@ -6,23 +6,36 @@ #pragma once #include "families/timm_senet/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmSENetImageClassificationPipeline final : public IImageClassification { +class TimmSENetImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: explicit TimmSENetImageClassificationPipeline(std::unique_ptr model, - TimmSENetPreprocessConfig preprocess_config = {}); - - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + TimmSENetPreprocessConfig preprocess_config, + std::int32_t num_classes, + std::string vocabulary_id, + std::vector labels); + + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmSENetPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_senet/runtime/plugin.cpp b/families/timm_senet/runtime/plugin.cpp index 03aefd5ef3..9328b68614 100644 --- a/families/timm_senet/runtime/plugin.cpp +++ b/families/timm_senet/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmSENetPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmSENetPreprocessConfig parse_config(const nlohmann::json& json) { TimmSENetPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -34,7 +33,7 @@ TimmSENetPreprocessConfig parse_config(const std::vector& data) { if (config.input_image_h <= 0 || config.input_image_w <= 0 || config.crop_pct <= 0.0F || config.crop_pct > 1.0F || config.image_mean.size() != 3 || config.image_std.size() != 3 || (config.interpolation != "bilinear" && config.interpolation != "bicubic")) { - throw std::runtime_error("timm SENet runtime.json does not match its contract"); + throw std::runtime_error("timm SENet runtime.json does not match its runtime contract"); } return config; } @@ -53,9 +52,13 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_senet/support.py b/families/timm_senet/support.py index a0102c0820..f9219f80a1 100644 --- a/families/timm_senet/support.py +++ b/families/timm_senet/support.py @@ -9,6 +9,6 @@ describe = family_support( model_types=("timm_senet", "senet154"), architectures=("senet154",), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_senet/tests/cpp/test_task_contract.cpp b/families/timm_senet/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..0aebe10ac5 --- /dev/null +++ b/families/timm_senet/tests/cpp/test_task_contract.cpp @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_senet/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{output_name, {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + std::string output_name{"logits"}; + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmSENetPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmSENetImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmSENetImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +void test_existing_named_output() { + auto module = std::make_unique(); + module->output_name = "class_scores"; + const auto expected = module->logits; + trtmc::TimmSENetImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const std::vector pixels(12, 0.75F); + require(model.run(request(pixels), {}).scores == expected, + "a single named classifier output retains the existing runtime contract"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + test_existing_named_output(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_senet/tests/manifests/senet154-gluon-in1k.json b/families/timm_senet/tests/manifests/senet154-gluon-in1k.json index ddb9d56f64..2a380561d7 100644 --- a/families/timm_senet/tests/manifests/senet154-gluon-in1k.json +++ b/families/timm_senet/tests/manifests/senet154-gluon-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "86f6d239ed30acbe3aca0cbb5769da2621a4e02c", "bundle": "senet154-gluon-in1k.bundle", "family": "timm_senet", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_senet/tests/performance.yaml b/families/timm_senet/tests/performance.yaml new file mode 100644 index 0000000000..41828503ba --- /dev/null +++ b/families/timm_senet/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-senet-performance +entries: + - id: timm_senet.classify + family: timm_senet + operation: classify + model: senet154-gluon-in1k + workload: + testcase: senet154-gluon-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_senet/tests/performance_reference.py b/families/timm_senet/tests/performance_reference.py new file mode 100644 index 0000000000..d8ba1cfd45 --- /dev/null +++ b/families/timm_senet/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time SENet inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model( + config["architecture"], pretrained=False, + pretrained_cfg=config["pretrained_cfg"], num_classes=int(config["num_classes"]), + checkpoint_path=str(checkpoint / "model.safetensors"), + ) + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("SENet must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("SENet returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_senet" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_senet image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_senet/tests/sdk_consumer.c b/families/timm_senet/tests/sdk_consumer.c new file mode 100644 index 0000000000..03243a7342 --- /dev/null +++ b/families/timm_senet/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "SENet must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "SENet must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_senet/tests/sdk_consumer.cpp b/families/timm_senet/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..5d4f52156b --- /dev/null +++ b/families/timm_senet/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("SENet must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("SENet must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_senet/tests/test_e2e.py b/families/timm_senet/tests/test_e2e.py index bd97e3ebfa..7a82fad023 100644 --- a/families/timm_senet/tests/test_e2e.py +++ b/families/timm_senet/tests/test_e2e.py @@ -8,6 +8,7 @@ from tools.e2e_evidence import evidence_stage, record_evidence import json +import math import os import subprocess from pathlib import Path @@ -28,7 +29,7 @@ def _cases() -> dict[str, tuple[dict, dict]]: for path in sorted(MANIFEST_ROOT.glob("*.json")): manifest = json.loads(path.read_text(encoding="utf-8")) assert manifest["family"] == FAMILY - assert manifest["task"] == "classification" + assert manifest["task"] == "image_to_class_scores" for case in manifest["testcases"]: name = str(case["name"]) assert name not in result @@ -105,6 +106,58 @@ def _asset(case: dict) -> Path: return path +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case)).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_senet_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: manifest, case = CASES[case_name] record_evidence("inputs", {"manifest": manifest, "case": CASES[case_name][-1]}) @@ -177,3 +230,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: ) with evidence_stage("compare"): assert int(actual["top_class"]) == int(np.argmax(expected)) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert len(actual["scores"]) == int(config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, int(np.argmax(expected)), tmp_path) diff --git a/families/timm_senet/tests/test_performance_reference.py b/families/timm_senet/tests/test_performance_reference.py new file mode 100644 index 0000000000..0b9022c568 --- /dev/null +++ b/families/timm_senet/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_senet.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_senet", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_senet.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_senet/tests/test_sdk_metadata.py b/families/timm_senet/tests/test_sdk_metadata.py new file mode 100644 index 0000000000..4eb934ce51 --- /dev/null +++ b/families/timm_senet/tests/test_sdk_metadata.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The builder publishes complete checkpoint-owned Task metadata.""" + +from types import SimpleNamespace + +import pytest + +from families.timm_senet import model +from tensorrt_model_connect import BuildRequest + + +@pytest.mark.parametrize("metadata,invalid", [ + ({}, False), + ({"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, False), + ({"label_names": ["only-one"]}, True), + ({"label_names": ["a", "b", "", "d", "e"]}, True), + ({"label_names": 5}, True), + ({"vocabulary_id": 5}, True), +]) +def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): + raw = {"num_classes": 5, **metadata} + runtime = {"image_height": 2, "image_width": 2, "num_classes": 5, + "crop_pct": 1.0, "interpolation": "bilinear", + "mean": [0.5] * 3, "std": [0.25] * 3} + monkeypatch.setattr(model, "_read_config", lambda _: raw) + monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) + monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + sections = {} + headers = [] + writer = SimpleNamespace(set_header=lambda **value: headers.append(value), + add_bytes=lambda key, value: sections.update({key: value}), + add_json=lambda key, value: sections.update({key: value})) + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_senet", task="image_to_class_scores", precision="fp32") + if invalid: + with pytest.raises(ValueError, match="vocabulary_id|label_names"): + model.build(request, writer) + assert not sections and not headers + return + model.build(request, writer) + assert headers == [{"family": "timm_senet", "task": "image_to_class_scores", "backend": "trt"}] + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) diff --git a/families/timm_seresnet/README.md b/families/timm_seresnet/README.md new file mode 100644 index 0000000000..335e683091 --- /dev/null +++ b/families/timm_seresnet/README.md @@ -0,0 +1,66 @@ +# timm SE-ResNet + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/seresnet50.a1_in1k -o model.bundle +trtmc classify model.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_seresnet_task_contract test_timm_seresnet_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_seresnet_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_seresnet_sdk_c model.bundle build image.rgb.f32 480 640 +build/test_timm_seresnet_sdk_cpp model.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_seresnet.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_seresnet/model.py b/families/timm_seresnet/model.py index 601a29b18f..7d252b5dcc 100644 --- a/families/timm_seresnet/model.py +++ b/families/timm_seresnet/model.py @@ -336,8 +336,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_seresnet does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_seresnet does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_seresnet supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_seresnet supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_seresnet does not support quantization") if request.fp32_layers: @@ -351,11 +351,23 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: str(request.precision).lower(), bool(request.verbose), ) + vocabulary_id = raw.get("vocabulary_id", "") + labels = raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm SE-ResNet vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != runtime["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm SE-ResNet label_names must name every class") writer.set_header(family="timm_seresnet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( "runtime.json", { + "num_classes": runtime["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "input_image_h": runtime["image_height"], "input_image_w": runtime["image_width"], "crop_pct": runtime["crop_pct"], diff --git a/families/timm_seresnet/runtime/CMakeLists.txt b/families/timm_seresnet/runtime/CMakeLists.txt index eaa905494c..432013a93d 100644 --- a/families/timm_seresnet/runtime/CMakeLists.txt +++ b/families/timm_seresnet/runtime/CMakeLists.txt @@ -35,6 +35,41 @@ install(TARGETS trtmc_model_timm_seresnet ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_seresnet_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_seresnet/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_seresnet_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_seresnet_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_seresnet_task_contract PRIVATE + trtmc_model_timm_seresnet trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_seresnet_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_seresnet_task_contract COMMAND test_timm_seresnet_task_contract) + set_tests_properties(timm_seresnet_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_seresnet_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_seresnet/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_seresnet_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_seresnet_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_seresnet_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_seresnet_task_contract + test_timm_seresnet_sdk_c test_timm_seresnet_sdk_cpp + ) + add_executable(test_timm_seresnet_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_seresnet/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_seresnet/runtime/pipeline.cpp b/families/timm_seresnet/runtime/pipeline.cpp index d927d62482..a015cb0705 100644 --- a/families/timm_seresnet/runtime/pipeline.cpp +++ b/families/timm_seresnet/runtime/pipeline.cpp @@ -5,51 +5,78 @@ #include "families/timm_seresnet/runtime/pipeline.h" -#include #include +#include #include #include namespace trtmc { + namespace { -const Tensor* find_logits(const TensorMap& outputs) { +const Tensor& require_logits(const TensorMap& outputs) { for (const auto& [name, tensor] : outputs) { - if (name.find("logits") != std::string::npos || outputs.size() == 1) - return &tensor; + if (name.find("logits") == std::string::npos && outputs.size() != 1) + continue; + if (tensor.data == nullptr || tensor.dtype != DType::kFloat32 || tensor.numel() <= 0) + throw std::runtime_error("timm SE-ResNet engine must return nonempty float32 logits"); + return tensor; } - return nullptr; + throw std::runtime_error("timm SE-ResNet engine did not return logits"); } } // namespace TimmSEResNetImageClassificationPipeline::TimmSEResNetImageClassificationPipeline( - std::unique_ptr model, TimmSEResNetPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmSEResNetPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmSEResNetImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm SE-ResNet class metadata does not match its output size"); } -ClassificationResult TimmSEResNetImageClassificationPipeline::classify(const float* pixels, - int32_t height, - int32_t width) { - auto values = preprocess_timm_seresnet_image(pixels, height, width, preprocess_config_); - Tensor input; - input.data = values.data(); - input.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; - input.dtype = DType::kFloat32; - const auto outputs = model_->forward({{"pixel_values", input}}); - const Tensor* logits = find_logits(outputs); - if (logits == nullptr || logits->numel() <= 0) - throw std::runtime_error("timm SEResNet engine returned no logits"); - if (logits->dtype != DType::kFloat32) - throw std::runtime_error("timm SEResNet logits must be float32"); - ClassificationResult result; - result.logits.resize(static_cast(logits->numel())); - std::memcpy(result.logits.data(), logits->data, result.logits.size() * sizeof(float)); - const auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = *best; +internal::LabelScoresResult +TimmSEResNetImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm SE-ResNet has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm SE-ResNet requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_seresnet_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); + + Tensor img_t; + img_t.data = pixel_values.data(); + img_t.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; + img_t.dtype = DType::kFloat32; + + auto outputs = model_->forward({{"pixel_values", img_t}}); + internal::LabelScoresResult result; + + const auto& logits_tensor = require_logits(outputs); + const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm SE-ResNet logits do not match its configured class count"); + + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, + static_cast(n) * sizeof(float)); + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_seresnet/runtime/pipeline.h b/families/timm_seresnet/runtime/pipeline.h index f66c24f61b..0695f0933c 100644 --- a/families/timm_seresnet/runtime/pipeline.h +++ b/families/timm_seresnet/runtime/pipeline.h @@ -6,23 +6,36 @@ #pragma once #include "families/timm_seresnet/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmSEResNetImageClassificationPipeline final : public IImageClassification { +class TimmSEResNetImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: - explicit TimmSEResNetImageClassificationPipeline( - std::unique_ptr model, TimmSEResNetPreprocessConfig preprocess_config = {}); - - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + explicit TimmSEResNetImageClassificationPipeline(std::unique_ptr model, + TimmSEResNetPreprocessConfig preprocess_config, + std::int32_t num_classes, + std::string vocabulary_id, + std::vector labels); + + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmSEResNetPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_seresnet/runtime/plugin.cpp b/families/timm_seresnet/runtime/plugin.cpp index 7cdf082df2..7fbe61a508 100644 --- a/families/timm_seresnet/runtime/plugin.cpp +++ b/families/timm_seresnet/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmSEResNetPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmSEResNetPreprocessConfig parse_config(const nlohmann::json& json) { TimmSEResNetPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -34,7 +33,7 @@ TimmSEResNetPreprocessConfig parse_config(const std::vector& data) { if (config.input_image_h <= 0 || config.input_image_w <= 0 || config.crop_pct <= 0.0F || config.crop_pct > 1.0F || config.image_mean.size() != 3 || config.image_std.size() != 3 || (config.interpolation != "bilinear" && config.interpolation != "bicubic")) { - throw std::runtime_error("timm SEResNet runtime.json does not match its contract"); + throw std::runtime_error("timm SE-ResNet runtime.json does not match its runtime contract"); } return config; } @@ -43,7 +42,7 @@ std::unique_ptr load_engine(IBackend& backend, const std::vectorok()) - throw std::runtime_error("timm SEResNet engine failed to load"); + throw std::runtime_error("timm SE-ResNet engine failed to load"); return engine; } @@ -53,9 +52,13 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_seresnet/support.py b/families/timm_seresnet/support.py index 3358f8da6b..6bb63c3f79 100644 --- a/families/timm_seresnet/support.py +++ b/families/timm_seresnet/support.py @@ -9,6 +9,6 @@ describe = family_support( model_types=("timm_seresnet", "seresnet50"), architectures=("seresnet50",), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_seresnet/tests/cpp/test_task_contract.cpp b/families/timm_seresnet/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..4683c2324d --- /dev/null +++ b/families/timm_seresnet/tests/cpp/test_task_contract.cpp @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_seresnet/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{output_name, {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + std::string output_name{"logits"}; + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmSEResNetPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmSEResNetImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "", {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmSEResNetImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +void test_existing_named_output() { + auto module = std::make_unique(); + module->output_name = "class_scores"; + const auto expected = module->logits; + trtmc::TimmSEResNetImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const std::vector pixels(12, 0.75F); + require(model.run(request(pixels), {}).scores == expected, + "a single named classifier output retains the existing runtime contract"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + test_existing_named_output(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_seresnet/tests/manifests/seresnet50-a1-in1k.json b/families/timm_seresnet/tests/manifests/seresnet50-a1-in1k.json index 352981ea5e..cc4dd1847e 100644 --- a/families/timm_seresnet/tests/manifests/seresnet50-a1-in1k.json +++ b/families/timm_seresnet/tests/manifests/seresnet50-a1-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "23f3482ee9acc4f51a2668a7cdf255fbd9420417", "bundle": "seresnet50-a1-in1k.bundle", "family": "timm_seresnet", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_seresnet/tests/performance.yaml b/families/timm_seresnet/tests/performance.yaml new file mode 100644 index 0000000000..d88d020c8a --- /dev/null +++ b/families/timm_seresnet/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-seresnet-performance +entries: + - id: timm_seresnet.classify + family: timm_seresnet + operation: classify + model: seresnet50-a1-in1k + workload: + testcase: seresnet50-a1-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_seresnet/tests/performance_reference.py b/families/timm_seresnet/tests/performance_reference.py new file mode 100644 index 0000000000..e7fabc2cb7 --- /dev/null +++ b/families/timm_seresnet/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time SE-ResNet inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model( + config["architecture"], pretrained=False, + pretrained_cfg=config["pretrained_cfg"], num_classes=int(config["num_classes"]), + checkpoint_path=str(checkpoint / "model.safetensors"), + ) + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("SE-ResNet must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("SE-ResNet returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_seresnet" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_seresnet image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_seresnet/tests/sdk_consumer.c b/families/timm_seresnet/tests/sdk_consumer.c new file mode 100644 index 0000000000..77036cadf0 --- /dev/null +++ b/families/timm_seresnet/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "SE-ResNet must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "SE-ResNet must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_seresnet/tests/sdk_consumer.cpp b/families/timm_seresnet/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..839d19d118 --- /dev/null +++ b/families/timm_seresnet/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("SE-ResNet must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("SE-ResNet must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_seresnet/tests/test_e2e.py b/families/timm_seresnet/tests/test_e2e.py index e0c657b53a..e67fe21629 100644 --- a/families/timm_seresnet/tests/test_e2e.py +++ b/families/timm_seresnet/tests/test_e2e.py @@ -8,6 +8,7 @@ from tools.e2e_evidence import evidence_stage, record_evidence import json +import math import os import subprocess from pathlib import Path @@ -28,7 +29,7 @@ def _cases() -> dict[str, tuple[dict, dict]]: for path in sorted(MANIFEST_ROOT.glob("*.json")): manifest = json.loads(path.read_text(encoding="utf-8")) assert manifest["family"] == FAMILY - assert manifest["task"] == "classification" + assert manifest["task"] == "image_to_class_scores" for case in manifest["testcases"]: name = str(case["name"]) assert name not in result @@ -105,6 +106,58 @@ def _asset(case: dict) -> Path: return path +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case)).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_seresnet_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: manifest, case = CASES[case_name] record_evidence("inputs", {"manifest": manifest, "case": CASES[case_name][-1]}) @@ -177,3 +230,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: ) with evidence_stage("compare"): assert int(actual["top_class"]) == int(np.argmax(expected)) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert len(actual["scores"]) == int(config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, int(np.argmax(expected)), tmp_path) diff --git a/families/timm_seresnet/tests/test_performance_reference.py b/families/timm_seresnet/tests/test_performance_reference.py new file mode 100644 index 0000000000..b32b123166 --- /dev/null +++ b/families/timm_seresnet/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_seresnet.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_seresnet", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_seresnet.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_seresnet/tests/test_sdk_metadata.py b/families/timm_seresnet/tests/test_sdk_metadata.py new file mode 100644 index 0000000000..37a1f63f1a --- /dev/null +++ b/families/timm_seresnet/tests/test_sdk_metadata.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The builder publishes complete checkpoint-owned Task metadata.""" + +from types import SimpleNamespace + +import pytest + +from families.timm_seresnet import model +from tensorrt_model_connect import BuildRequest + + +@pytest.mark.parametrize("metadata,invalid", [ + ({}, False), + ({"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, False), + ({"label_names": ["only-one"]}, True), + ({"label_names": ["a", "b", "", "d", "e"]}, True), + ({"label_names": 5}, True), + ({"vocabulary_id": 5}, True), +]) +def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): + raw = {"num_classes": 5, **metadata} + runtime = {"image_height": 2, "image_width": 2, "num_classes": 5, + "crop_pct": 1.0, "interpolation": "bilinear", + "mean": [0.5] * 3, "std": [0.25] * 3} + monkeypatch.setattr(model, "_read_config", lambda _: raw) + monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) + monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + sections = {} + headers = [] + writer = SimpleNamespace(set_header=lambda **value: headers.append(value), + add_bytes=lambda key, value: sections.update({key: value}), + add_json=lambda key, value: sections.update({key: value})) + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_seresnet", task="image_to_class_scores", precision="fp32") + if invalid: + with pytest.raises(ValueError, match="vocabulary_id|label_names"): + model.build(request, writer) + assert not sections and not headers + return + model.build(request, writer) + assert headers == [{"family": "timm_seresnet", "task": "image_to_class_scores", "backend": "trt"}] + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) diff --git a/families/timm_vgg/README.md b/families/timm_vgg/README.md new file mode 100644 index 0000000000..aa34335531 --- /dev/null +++ b/families/timm_vgg/README.md @@ -0,0 +1,66 @@ +# timm VGG + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/vgg16.tv_in1k -o model.bundle +trtmc classify model.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_vgg_task_contract test_timm_vgg_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_vgg_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_vgg_sdk_c model.bundle build image.rgb.f32 480 640 +build/test_timm_vgg_sdk_cpp model.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_vgg.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_vgg/model.py b/families/timm_vgg/model.py index 156411d43a..d31c9c9a14 100644 --- a/families/timm_vgg/model.py +++ b/families/timm_vgg/model.py @@ -262,10 +262,21 @@ def build_engine( def get_bundle_config_overrides(self, config: ModelConfig) -> dict: cfg = config.raw.get("_timm_vgg_config") or _resolve_vgg_config(config.raw) + vocabulary_id = config.raw.get("vocabulary_id", "") + labels = config.raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm VGG vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != cfg["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm VGG label_names must name every class") return { "input_image_h": cfg["image_size_h"], "input_image_w": cfg["image_size_w"], "num_classes": cfg["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "image_mean": cfg["mean"], "image_std": cfg["std"], "crop_pct": cfg["crop_pct"], @@ -290,8 +301,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_vgg does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_vgg does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_vgg supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_vgg supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_vgg does not support quantization") if request.fp32_layers: @@ -313,14 +324,17 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: precision=precision, verbose=bool(request.verbose), ) + runtime_source = model.get_bundle_config_overrides(config) writer.set_header(family="timm_vgg", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) - runtime_source = model.get_bundle_config_overrides(config) writer.add_json( "runtime.json", { key: runtime_source[key] for key in ( + "num_classes", + "vocabulary_id", + "labels", "input_image_h", "input_image_w", "crop_pct", diff --git a/families/timm_vgg/runtime/CMakeLists.txt b/families/timm_vgg/runtime/CMakeLists.txt index 8154dcdb4a..946dd96262 100644 --- a/families/timm_vgg/runtime/CMakeLists.txt +++ b/families/timm_vgg/runtime/CMakeLists.txt @@ -38,6 +38,41 @@ install(TARGETS trtmc_model_timm_vgg ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_vgg_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_vgg/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_vgg_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_vgg_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_vgg_task_contract PRIVATE + trtmc_model_timm_vgg trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_vgg_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_vgg_task_contract COMMAND test_timm_vgg_task_contract) + set_tests_properties(timm_vgg_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_vgg_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_vgg/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_vgg_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_vgg_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_vgg_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_vgg_task_contract + test_timm_vgg_sdk_c test_timm_vgg_sdk_cpp + ) + add_executable(test_timm_vgg_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_vgg/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_vgg/runtime/pipeline.cpp b/families/timm_vgg/runtime/pipeline.cpp index dcf91fc411..e13a79e77e 100644 --- a/families/timm_vgg/runtime/pipeline.cpp +++ b/families/timm_vgg/runtime/pipeline.cpp @@ -5,8 +5,8 @@ #include "families/timm_vgg/runtime/pipeline.h" -#include #include +#include #include #include @@ -18,23 +18,44 @@ const Tensor& require_logits(const TensorMap& outputs) { const auto found = outputs.find("logits"); if (found == outputs.end()) throw std::runtime_error("timm VGG engine did not return logits"); - if (found->second.numel() <= 0) - throw std::runtime_error("timm VGG engine returned empty logits"); + if (found->second.data == nullptr || found->second.dtype != DType::kFloat32 || + found->second.numel() <= 0) + throw std::runtime_error("timm VGG engine must return nonempty float32 logits"); return found->second; } } // namespace TimmVggImageClassificationPipeline::TimmVggImageClassificationPipeline( - std::unique_ptr model, TimmVggPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmVggPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmVggImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm VGG class metadata does not match its output size"); } -ClassificationResult TimmVggImageClassificationPipeline::classify(const float* pixels, - int32_t height, int32_t width) { - auto pixel_values = preprocess_timm_vgg_image(pixels, height, width, preprocess_config_); +internal::LabelScoresResult +TimmVggImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm VGG has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm VGG requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_vgg_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); Tensor img_t; img_t.data = pixel_values.data(); @@ -42,18 +63,19 @@ ClassificationResult TimmVggImageClassificationPipeline::classify(const float* p img_t.dtype = DType::kFloat32; auto outputs = model_->forward({{"pixel_values", img_t}}); - ClassificationResult result; + internal::LabelScoresResult result; const auto& logits_tensor = require_logits(outputs); const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm VGG logits do not match its configured class count"); - result.logits.resize(static_cast(n)); - std::memcpy(result.logits.data(), logits_tensor.data, + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, static_cast(n) * sizeof(float)); - - auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = (best == result.logits.end()) ? 0.0F : *best; + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_vgg/runtime/pipeline.h b/families/timm_vgg/runtime/pipeline.h index c2198df536..f153a5653d 100644 --- a/families/timm_vgg/runtime/pipeline.h +++ b/families/timm_vgg/runtime/pipeline.h @@ -6,23 +6,35 @@ #pragma once #include "families/timm_vgg/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmVggImageClassificationPipeline final : public IImageClassification { +class TimmVggImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: explicit TimmVggImageClassificationPipeline(std::unique_ptr model, - TimmVggPreprocessConfig preprocess_config); + TimmVggPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, + std::vector labels); - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmVggPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_vgg/runtime/plugin.cpp b/families/timm_vgg/runtime/plugin.cpp index a3c54ab4d8..d439b7dc43 100644 --- a/families/timm_vgg/runtime/plugin.cpp +++ b/families/timm_vgg/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmVggPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmVggPreprocessConfig parse_config(const nlohmann::json& json) { TimmVggPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -55,7 +54,11 @@ extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context throw std::invalid_argument("timm_vgg does not support --kv-cache-size"); const auto& config_data = trtmc::timm_vgg::require_section(context.reader, "runtime.json"); const auto& plan = trtmc::timm_vgg::require_section(context.reader, "engine.plan"); - auto config = trtmc::timm_vgg::parse_config(config_data); + const auto metadata = nlohmann::json::parse(config_data.begin(), config_data.end()); + auto config = trtmc::timm_vgg::parse_config(metadata); auto engine = trtmc::timm_vgg::load_engine(context.backend, plan); - return new trtmc::TimmVggImageClassificationPipeline(std::move(engine), std::move(config)); + return new trtmc::TimmVggImageClassificationPipeline( + std::move(engine), std::move(config), metadata.at("num_classes").get(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_vgg/support.py b/families/timm_vgg/support.py index 7a65c2d0a1..c5aaf44040 100644 --- a/families/timm_vgg/support.py +++ b/families/timm_vgg/support.py @@ -9,6 +9,6 @@ describe = family_support( model_types=("timm_vgg",), architectures=("vgg11", "vgg13", "vgg16", "vgg19"), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_vgg/tests/cpp/test_task_contract.cpp b/families/timm_vgg/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..3d2f21aac2 --- /dev/null +++ b/families/timm_vgg/tests/cpp/test_task_contract.cpp @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_vgg/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{"logits", {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmVggPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmVggImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmVggImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_vgg/tests/manifests/vgg16-tv-in1k.json b/families/timm_vgg/tests/manifests/vgg16-tv-in1k.json index 6e94e1192e..2fc6fe719f 100644 --- a/families/timm_vgg/tests/manifests/vgg16-tv-in1k.json +++ b/families/timm_vgg/tests/manifests/vgg16-tv-in1k.json @@ -3,7 +3,7 @@ "hf_id": "timm/vgg16.tv_in1k", "bundle": "vgg16-tv-in1k.bundle", "family": "timm_vgg", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_vgg/tests/performance.yaml b/families/timm_vgg/tests/performance.yaml new file mode 100644 index 0000000000..ec108b12c7 --- /dev/null +++ b/families/timm_vgg/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-vgg-performance +entries: + - id: timm_vgg.classify + family: timm_vgg + operation: classify + model: vgg16-tv-in1k + workload: + testcase: vgg16-tv-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_vgg/tests/performance_reference.py b/families/timm_vgg/tests/performance_reference.py new file mode 100644 index 0000000000..7dd8aa4d60 --- /dev/null +++ b/families/timm_vgg/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time VGG inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from safetensors.torch import load_file + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model(config["architecture"], pretrained=False, + num_classes=int(config["num_classes"])) + model.load_state_dict(load_file(str(checkpoint / "model.safetensors")), strict=True) + model.pretrained_cfg = config["pretrained_cfg"] + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("VGG must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("VGG returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_vgg" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_vgg image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_vgg/tests/sdk_consumer.c b/families/timm_vgg/tests/sdk_consumer.c new file mode 100644 index 0000000000..55d5df5943 --- /dev/null +++ b/families/timm_vgg/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "VGG must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "VGG must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_vgg/tests/sdk_consumer.cpp b/families/timm_vgg/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..f73689f61e --- /dev/null +++ b/families/timm_vgg/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("VGG must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("VGG must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_vgg/tests/test_e2e.py b/families/timm_vgg/tests/test_e2e.py index 7c4f1fa2ee..3986353981 100644 --- a/families/timm_vgg/tests/test_e2e.py +++ b/families/timm_vgg/tests/test_e2e.py @@ -7,6 +7,7 @@ from tools.e2e_evidence import evidence_stage, record_evidence import json +import math import os import subprocess from pathlib import Path @@ -15,7 +16,7 @@ from tensorrt_model_connect import BuildRequest, build FAMILY = "timm_vgg" -TASKS = frozenset({"classification"}) +TASKS = frozenset({"image_to_class_scores"}) TEST_ROOT = Path(__file__).resolve().parent MANIFEST_ROOT = TEST_ROOT / "manifests" @@ -249,6 +250,58 @@ def _assert_parity(actual, expected) -> None: assert int(np.argmax(actual["logits"])) == int(np.argmax(expected["logits"])) +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case["test_image"])).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_vgg_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: _, manifest, case = CASES[case_name] record_evidence("inputs", {"manifest": manifest, "case": CASES[case_name][-1]}) @@ -267,3 +320,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: record_evidence("reference", {**expected, "top_class": int(np.argmax(expected["logits"]))}) with evidence_stage("compare"): _assert_parity(actual, expected) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert len(actual["scores"]) == int(config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, int(np.argmax(expected["logits"])), tmp_path) diff --git a/families/timm_vgg/tests/test_model.py b/families/timm_vgg/tests/test_model.py index 5ecfd0ba9a..72525dbba7 100644 --- a/families/timm_vgg/tests/test_model.py +++ b/families/timm_vgg/tests/test_model.py @@ -178,7 +178,7 @@ def test_build_rejects_batch_norm_variant_the_graph_does_not_implement(tmp_path: model_dir=tmp_path, output_path=tmp_path / "unused.bundle", family="timm_vgg", - task="classification", + task="image_to_class_scores", precision="fp16", max_sequence_length=1, ) @@ -192,7 +192,7 @@ def test_build_rejects_quantization(tmp_path: Path): model_dir=tmp_path, output_path=tmp_path / "unused.bundle", family="timm_vgg", - task="classification", + task="image_to_class_scores", precision="fp16", quantization="fp8", ) @@ -209,3 +209,48 @@ def test_build_engine_rejects_input_not_divisible_by_the_pool_count(tmp_path: Pa with pytest.raises(ValueError, match="divisible by 8"): model.build_engine(cfg, weights, precision="fp32") + + +@pytest.mark.parametrize("metadata", [ + {}, + {"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, +]) +def test_build_exports_task_and_checkpoint_metadata(tmp_path, monkeypatch, metadata): + _write_tiny_vgg(tmp_path) + raw = json.loads((tmp_path / "config.json").read_text()) + raw.update(metadata) + (tmp_path / "config.json").write_text(json.dumps(raw)) + monkeypatch.setattr(_TimmVggModel, "build_engine", lambda *args, **kwargs: b"plan") + sections = {} + + class Writer: + def set_header(self, **value): + sections["header"] = value + + def add_bytes(self, name, value): + sections[name] = value + + def add_json(self, name, value): + sections[name] = value + + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_vgg", task="image_to_class_scores", precision="fp32") + build_family(request, Writer()) + assert sections["header"]["task"] == "image_to_class_scores" + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + + +@pytest.mark.parametrize("labels", [["only-one"], ["a", "b", "", "d", "e"], 5]) +def test_build_rejects_incomplete_class_labels(tmp_path, monkeypatch, labels): + _write_tiny_vgg(tmp_path) + raw = json.loads((tmp_path / "config.json").read_text()) + raw["label_names"] = labels + (tmp_path / "config.json").write_text(json.dumps(raw)) + monkeypatch.setattr(_TimmVggModel, "build_engine", lambda *args, **kwargs: b"plan") + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_vgg", task="image_to_class_scores", precision="fp32") + with pytest.raises(ValueError, match="label_names must name every class"): + build_family(request, object()) diff --git a/families/timm_vgg/tests/test_performance_reference.py b/families/timm_vgg/tests/test_performance_reference.py new file mode 100644 index 0000000000..7bd20aeff5 --- /dev/null +++ b/families/timm_vgg/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_vgg.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_vgg", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_vgg.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_xception/README.md b/families/timm_xception/README.md new file mode 100644 index 0000000000..c211f96524 --- /dev/null +++ b/families/timm_xception/README.md @@ -0,0 +1,66 @@ +# timm Xception + +This family implements `image_to_class_scores` through `IImageToClassScores`. +The family binds the interface on its loaded model; the shared C ABI and the +header-only C++ `ImageToClassScores` wrapper discover that binding without a +family-specific shared registry. + +The input is contiguous host RGB float32 in `[0, 1]`. The family owns the +checkpoint's resize, crop, normalization and engine call. The output contains +every class logit in checkpoint order, without softmax or top-k truncation. +`label_names` and `vocabulary_id` are retained when explicitly supplied by the +checkpoint. Missing metadata is returned as empty labels/identity: the class +indices remain model-local ordinals. There are no runtime Config options; +unsupported overrides are rejected. + +Build a new bundle with this family version; the old `classification` bundle +mode is not retained. The existing CLI command remains simple: + +```sh +trtmc build timm/xception41.tf_in1k -o model.bundle +trtmc classify model.bundle --image photo.jpg +``` + +## Validation + +The existing official-checkpoint test still compares the native top-1 class +against the timm reference using the original image and preprocessing. It also +checks the complete logits and class metadata exposed by the semantic Task. +That same E2E executes both public SDK consumers, using the existing +`TRTMC_NATIVE_BUILD_DIR` to locate their built binaries. They receive identical +decoded RGB input, must agree on every score, and retain the original top-1 +reference check. No new CI selector or environment variable is introduced. + +The CPU contract test checks binding, unknown and explicit class identity, +normalization, complete owned output, and rejection before inference of invalid +input or unsupported Config. It also builds the two public SDK consumers: + +```sh +cmake --build build --target test_timm_xception_task_contract test_timm_xception_image_preprocess +ctest --test-dir build --output-on-failure -R '^timm_xception_(task_contract|image_preprocess)$' +``` + +`tests/sdk_consumer.c` and `tests/sdk_consumer.cpp` use only the public SDK. For +an already-built bundle, supply an unprocessed RGB float32 HWC file and its +original height and width: + +```sh +build/test_timm_xception_sdk_c model.bundle build image.rgb.f32 480 640 +build/test_timm_xception_sdk_cpp model.bundle build image.rgb.f32 480 640 +``` + +Each prints all scores as JSON and reads the result after releasing its model +handle. These commands perform real inference and require the matching runtime +and checkpoint bundle; the CPU contract test alone does not qualify a checkpoint. + +## Benchmark timing + +`tests/performance.yaml` takes over the existing `timm_xception.classify` +entry through the benchmark's family-owned reference protocol. The workload, +precision, 3 warmups, 10 measurements, 5% margin and top-class oracle are unchanged. +The reference times inference, complete float32 host logits and synchronization; +argmax, finite checks and JSON reporting happen after timing, as in the semantic +SDK benchmark worker. The existing reference policy still excludes input +preparation (`task-model-call-wall`), while the native public Task call includes +family preprocessing. This fixes reduction/reporting placement, not that existing +scope difference, and does not establish a performance improvement. diff --git a/families/timm_xception/model.py b/families/timm_xception/model.py index 2a6c1eaaf0..4c404569c9 100644 --- a/families/timm_xception/model.py +++ b/families/timm_xception/model.py @@ -353,8 +353,8 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_xception does not support tensor parallelism") if request.context_parallel_size != 1: raise NotImplementedError("timm_xception does not support context parallelism") - if request.task != "classification": - raise ValueError("timm_xception supports only task=classification") + if request.task != "image_to_class_scores": + raise ValueError("timm_xception supports only task=image_to_class_scores") if request.quantization not in {None, "none"}: raise NotImplementedError("timm_xception does not support quantization") if request.fp32_layers: @@ -368,11 +368,23 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: str(request.precision).lower(), bool(request.verbose), ) + vocabulary_id = raw.get("vocabulary_id", "") + labels = raw.get("label_names", []) + if not isinstance(vocabulary_id, str): + raise ValueError("timm Xception vocabulary_id must be a string") + if not isinstance(labels, list) or (labels and ( + len(labels) != runtime["num_classes"] + or any(not isinstance(label, str) or not label for label in labels) + )): + raise ValueError("timm Xception label_names must name every class") writer.set_header(family="timm_xception", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( "runtime.json", { + "num_classes": runtime["num_classes"], + "vocabulary_id": vocabulary_id, + "labels": labels, "input_image_h": runtime["image_height"], "input_image_w": runtime["image_width"], "crop_pct": runtime["crop_pct"], diff --git a/families/timm_xception/runtime/CMakeLists.txt b/families/timm_xception/runtime/CMakeLists.txt index 5a5b46223d..895339ad8a 100644 --- a/families/timm_xception/runtime/CMakeLists.txt +++ b/families/timm_xception/runtime/CMakeLists.txt @@ -35,6 +35,41 @@ install(TARGETS trtmc_model_timm_xception ) if(TRTMC_BUILD_TESTS) + add_executable(test_timm_xception_task_contract + ${PROJECT_SOURCE_DIR}/families/timm_xception/tests/cpp/test_task_contract.cpp + ) + target_include_directories(test_timm_xception_task_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_include_directories(test_timm_xception_task_contract SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_timm_xception_task_contract PRIVATE + trtmc_model_timm_xception trtmc_core ${TRTMC_CUDART_LIBRARY} + ) + target_compile_options(test_timm_xception_task_contract PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + add_test(NAME timm_xception_task_contract COMMAND test_timm_xception_task_contract) + set_tests_properties(timm_xception_task_contract PROPERTIES LABELS "cpu") + + foreach(_language IN ITEMS c cpp) + add_executable(test_timm_xception_sdk_${_language} + ${PROJECT_SOURCE_DIR}/families/timm_xception/tests/sdk_consumer.${_language} + ) + target_link_libraries(test_timm_xception_sdk_${_language} PRIVATE trtmc_c) + target_compile_options(test_timm_xception_sdk_${_language} PRIVATE + -Wall -Wextra -Wpedantic -Werror + ) + set_target_properties(test_timm_xception_sdk_${_language} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + endforeach() + add_dependencies(test_timm_xception_task_contract + test_timm_xception_sdk_c test_timm_xception_sdk_cpp + ) + add_executable(test_timm_xception_image_preprocess ${PROJECT_SOURCE_DIR}/families/timm_xception/tests/cpp/test_image_preprocess_seam.cpp ) diff --git a/families/timm_xception/runtime/pipeline.cpp b/families/timm_xception/runtime/pipeline.cpp index 68ee6569f5..a6048203c2 100644 --- a/families/timm_xception/runtime/pipeline.cpp +++ b/families/timm_xception/runtime/pipeline.cpp @@ -5,51 +5,78 @@ #include "families/timm_xception/runtime/pipeline.h" -#include #include +#include #include #include namespace trtmc { + namespace { -const Tensor* find_logits(const TensorMap& outputs) { +const Tensor& require_logits(const TensorMap& outputs) { for (const auto& [name, tensor] : outputs) { - if (name.find("logits") != std::string::npos || outputs.size() == 1) - return &tensor; + if (name.find("logits") == std::string::npos && outputs.size() != 1) + continue; + if (tensor.data == nullptr || tensor.dtype != DType::kFloat32 || tensor.numel() <= 0) + throw std::runtime_error("timm Xception engine must return nonempty float32 logits"); + return tensor; } - return nullptr; + throw std::runtime_error("timm Xception engine did not return logits"); } } // namespace TimmXceptionImageClassificationPipeline::TimmXceptionImageClassificationPipeline( - std::unique_ptr model, TimmXceptionPreprocessConfig preprocess_config) - : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)) { + std::unique_ptr model, TimmXceptionPreprocessConfig preprocess_config, + std::int32_t num_classes, std::string vocabulary_id, std::vector labels) + : model_(std::move(model)), preprocess_config_(std::move(preprocess_config)), + num_classes_(num_classes), vocabulary_id_(std::move(vocabulary_id)), + labels_(std::move(labels)) { if (!model_ || !model_->ok()) throw std::runtime_error("TimmXceptionImageClassificationPipeline: invalid model"); + if (num_classes_ <= 0 || + (!labels_.empty() && labels_.size() != static_cast(num_classes_))) + throw std::runtime_error("timm Xception class metadata does not match its output size"); } -ClassificationResult TimmXceptionImageClassificationPipeline::classify(const float* pixels, - int32_t height, - int32_t width) { - auto values = preprocess_timm_xception_image(pixels, height, width, preprocess_config_); - Tensor input; - input.data = values.data(); - input.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; - input.dtype = DType::kFloat32; - const auto outputs = model_->forward({{"pixel_values", input}}); - const Tensor* logits = find_logits(outputs); - if (logits == nullptr || logits->numel() <= 0) - throw std::runtime_error("timm Xception engine returned no logits"); - if (logits->dtype != DType::kFloat32) - throw std::runtime_error("timm Xception logits must be float32"); - ClassificationResult result; - result.logits.resize(static_cast(logits->numel())); - std::memcpy(result.logits.data(), logits->data, result.logits.size() * sizeof(float)); - const auto best = std::max_element(result.logits.begin(), result.logits.end()); - result.top_class = static_cast(std::distance(result.logits.begin(), best)); - result.top_score = *best; +internal::LabelScoresResult +TimmXceptionImageClassificationPipeline::run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) { + if (!config.empty()) + throw internal::ConfigError("timm Xception has no runtime configuration"); + const auto& image = request.image; + if (image.format != internal::ImageFormat::Float32 || image.channels != 3 || + image.data == nullptr || image.height == 0 || image.width == 0 || + image.height > static_cast(std::numeric_limits::max()) || + image.width > static_cast(std::numeric_limits::max()) || + static_cast(image.height) > + std::numeric_limits::max() / image.width / 3 / sizeof(float) || + image.byte_size != static_cast(image.height) * image.width * 3 * sizeof(float)) + throw std::invalid_argument("timm Xception requires contiguous float32 RGB input"); + auto pixel_values = preprocess_timm_xception_image( + static_cast(image.data), static_cast(image.height), + static_cast(image.width), preprocess_config_); + + Tensor img_t; + img_t.data = pixel_values.data(); + img_t.shape = {1, 3, preprocess_config_.input_image_h, preprocess_config_.input_image_w}; + img_t.dtype = DType::kFloat32; + + auto outputs = model_->forward({{"pixel_values", img_t}}); + internal::LabelScoresResult result; + + const auto& logits_tensor = require_logits(outputs); + const auto n = logits_tensor.numel(); + if (n != static_cast(num_classes_)) + throw std::runtime_error("timm Xception logits do not match its configured class count"); + + result.scores.resize(static_cast(n)); + std::memcpy(result.scores.data(), logits_tensor.data, + static_cast(n) * sizeof(float)); + result.kind = internal::ScoreKind::Logit; + result.vocabulary_id = vocabulary_id_; + result.labels = labels_; return result; } diff --git a/families/timm_xception/runtime/pipeline.h b/families/timm_xception/runtime/pipeline.h index 766d67c8a9..0551284c3c 100644 --- a/families/timm_xception/runtime/pipeline.h +++ b/families/timm_xception/runtime/pipeline.h @@ -6,23 +6,36 @@ #pragma once #include "families/timm_xception/runtime/image_preprocess_seam.h" +#include "trtmc/internal/features.h" +#include "trtmc/internal/model.h" #include "trtmc/runtime/trt_module.h" -#include "trtmc/task.h" #include namespace trtmc { -class TimmXceptionImageClassificationPipeline final : public IImageClassification { +class TimmXceptionImageClassificationPipeline final : public internal::IModel, + public internal::IImageToClassScores { public: - explicit TimmXceptionImageClassificationPipeline( - std::unique_ptr model, TimmXceptionPreprocessConfig preprocess_config = {}); - - ClassificationResult classify(const float* pixels, int32_t height, int32_t width) override; + explicit TimmXceptionImageClassificationPipeline(std::unique_ptr model, + TimmXceptionPreprocessConfig preprocess_config, + std::int32_t num_classes, + std::string vocabulary_id, + std::vector labels); + + const char* task() const noexcept override { return IImageToClassScores::kTask.data(); } + std::vector task_bindings() override { + return {internal::bind(*this)}; + } + internal::LabelScoresResult run(const internal::ImageToClassScoresRequest& request, + internal::ConfigView config) override; private: std::unique_ptr model_; TimmXceptionPreprocessConfig preprocess_config_; + std::int32_t num_classes_; + std::string vocabulary_id_; + std::vector labels_; }; } // namespace trtmc diff --git a/families/timm_xception/runtime/plugin.cpp b/families/timm_xception/runtime/plugin.cpp index 87ce620d69..3d48c6dc31 100644 --- a/families/timm_xception/runtime/plugin.cpp +++ b/families/timm_xception/runtime/plugin.cpp @@ -22,8 +22,7 @@ std::vector require_section(const BundleReader& bundle, const char* name) return bundle.read_section(name); } -TimmXceptionPreprocessConfig parse_config(const std::vector& data) { - const auto json = nlohmann::json::parse(data.begin(), data.end()); +TimmXceptionPreprocessConfig parse_config(const nlohmann::json& json) { TimmXceptionPreprocessConfig config; config.input_image_h = json.at("input_image_h").get(); config.input_image_w = json.at("input_image_w").get(); @@ -34,7 +33,7 @@ TimmXceptionPreprocessConfig parse_config(const std::vector& data) { if (config.input_image_h <= 0 || config.input_image_w <= 0 || config.crop_pct <= 0.0F || config.crop_pct > 1.0F || config.image_mean.size() != 3 || config.image_std.size() != 3 || (config.interpolation != "bilinear" && config.interpolation != "bicubic")) { - throw std::runtime_error("timm Xception runtime.json does not match its contract"); + throw std::runtime_error("timm Xception runtime.json does not match its runtime contract"); } return config; } @@ -53,9 +52,13 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector(), + metadata.at("vocabulary_id").get(), + metadata.at("labels").get>()); } diff --git a/families/timm_xception/support.py b/families/timm_xception/support.py index 1941f3a4fb..a06f39630a 100644 --- a/families/timm_xception/support.py +++ b/families/timm_xception/support.py @@ -9,6 +9,6 @@ describe = family_support( model_types=("timm_xception", "xception41"), architectures=("xception41",), - tasks=("classification",), - default_task="classification", + tasks=("image_to_class_scores",), + default_task="image_to_class_scores", ) diff --git a/families/timm_xception/tests/cpp/test_task_contract.cpp b/families/timm_xception/tests/cpp/test_task_contract.cpp new file mode 100644 index 0000000000..0047fb64b3 --- /dev/null +++ b/families/timm_xception/tests/cpp/test_task_contract.cpp @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/timm_xception/runtime/pipeline.h" + +#include +#include +#include + +namespace { + +class RecordingModule final : public trtmc::ITrtModule { + public: + trtmc::TensorMap forward(const trtmc::TensorMap& inputs) override { + ++calls; + const auto& image = inputs.at("pixel_values"); + shape = image.shape; + const auto* data = static_cast(image.data); + pixels.assign(data, data + image.numel()); + return {{output_name, {logits.data(), {1, static_cast(logits.size())}, dtype}}}; + } + trtmc::DeviceTensorMap forward_device(const trtmc::DeviceTensorMap&) override { return {}; } + void forward_device_async(const trtmc::DeviceTensorMap&) override {} + void forward_async(const trtmc::TensorMap&) override {} + void sync() override {} + cudaStream_t stream() const override { return nullptr; } + void enable_cuda_graph() override {} + bool cuda_graph_active() const override { return false; } + bool cuda_graph_captured() const override { return false; } + int32_t profile_idx() const override { return 0; } + std::vector input_info() const override { return {}; } + std::vector output_info() const override { return {}; } + bool has_input(const std::string& name) const override { return name == "pixel_values"; } + bool has_output(const std::string& name) const override { return name == "logits"; } + trtmc::DType tensor_dtype(const std::string&) const override { return dtype; } + std::vector tensor_shape(const std::string&) const override { return {}; } + std::vector input_profile_shape(const std::string&, int32_t, + trtmc::ProfileShapeSelector) const override { + return {}; + } + int32_t optimization_profile_count() const override { return 1; } + void* device_ptr(const std::string&) const override { return nullptr; } + void bind_external(const std::string&, void*) override {} + void bind_external(const std::string&, void*, const std::vector&) override {} + int32_t input_rank(const std::string&) const override { return 4; } + bool input_is_dynamic(const std::string&) const override { return false; } + void reset_execution_context() override {} + void set_timing_label(std::string) override {} + bool ok() const override { return true; } + void keep_alive(std::shared_ptr) override {} + + std::string output_name{"logits"}; + int calls{0}; + std::vector logits{-2.0F, 4.0F, 0.5F, 3.0F, -1.0F}; + std::vector pixels; + std::vector shape; + trtmc::DType dtype{trtmc::DType::kFloat32}; +}; + +void require(bool value, const char* message) { + if (!value) + throw std::runtime_error(message); +} + +template +void rejects(Function function, const char* message) { + try { + function(); + } catch (const Error&) { + return; + } + throw std::runtime_error(message); +} + +trtmc::TimmXceptionPreprocessConfig preprocessing() { + return {2, 2, {0.25F, 0.5F, 0.75F}, {0.5F, 0.25F, 0.125F}, 1.0F, "bilinear"}; +} + +trtmc::internal::ImageToClassScoresRequest request(const std::vector& pixels) { + return {{pixels.data(), pixels.size() * sizeof(float), 2, 2, 3, + trtmc::internal::ImageFormat::Float32}}; +} + +void test_binding_and_complete_owned_logits() { + trtmc::internal::LabelScoresResult result; + { + auto module = std::make_unique(); + auto* recording = module.get(); + trtmc::TimmXceptionImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "", {}); + const auto bindings = model.task_bindings(); + require(bindings.size() == 1 && bindings[0].key.id == "image_to_class_scores" && + bindings[0].key.major == 1 && bindings[0].key.minor == 0, + "family must publish exactly its implemented semantic task"); + require(bindings[0].fields.empty(), "family has no runtime config options"); + require(bindings[0].implementation == + static_cast(&model), + "bind must preserve the interface subobject address"); + require(std::string(model.task()) == "image_to_class_scores", + "bundle primary task must match the binding"); + const std::vector pixels(12, 0.75F); + result = static_cast(bindings[0].implementation) + ->run(request(pixels), {}); + require(result.scores == recording->logits, "return all raw logits in class order"); + require(result.kind == trtmc::internal::ScoreKind::Logit, "scores are logits"); + require(result.labels.empty() && result.vocabulary_id.empty(), + "unknown vocabulary must not acquire invented identity or labels"); + require(recording->shape == std::vector({1, 3, 2, 2}), + "preprocessing must preserve the engine NCHW input layout"); + require(recording->pixels == std::vector({1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0}), + "preprocessing must preserve checkpoint normalization"); + recording->logits.assign(5, 0.0F); + model.run(request(pixels), {}); + } + require(result.scores == std::vector({-2, 4, 0.5F, 3, -1}), + "result must own all logits across another call and model destruction"); +} + +void test_metadata_and_invalid_inputs() { + auto module = std::make_unique(); + auto* recording = module.get(); + const std::vector labels{"first", "second", "third", "fourth", "fifth"}; + trtmc::TimmXceptionImageClassificationPipeline model(std::move(module), preprocessing(), 5, + "test:five-classes", labels); + const std::vector pixels(12, 0.75F); + const auto input = request(pixels); + const auto result = model.run(input, {}); + require(result.vocabulary_id == "test:five-classes" && result.labels == labels, + "return checkpoint-provided identity without reordering"); + + const trtmc::internal::ConfigEntry unsupported{"top_k", std::int64_t{1}}; + rejects([&] { model.run(input, {&unsupported, 1}); }, + "unsupported config must fail, not truncate logits"); + auto invalid = input; + invalid.image.byte_size -= sizeof(float); + rejects([&] { model.run(invalid, {}); }, "reject truncated image"); + invalid = input; + invalid.image.channels = 4; + rejects([&] { model.run(invalid, {}); }, "reject non-RGB image"); + invalid = input; + invalid.image.format = trtmc::internal::ImageFormat::UInt8; + rejects([&] { model.run(invalid, {}); }, "reject unsupported format"); + require(recording->calls == 1, "invalid input and config must fail before engine execution"); + recording->logits.pop_back(); + rejects([&] { model.run(input, {}); }, "reject incomplete class output"); + recording->logits.push_back(0.0F); + recording->dtype = trtmc::DType::kFloat16; + rejects([&] { model.run(input, {}); }, "reject wrong output dtype"); +} + +void test_existing_named_output() { + auto module = std::make_unique(); + module->output_name = "class_scores"; + const auto expected = module->logits; + trtmc::TimmXceptionImageClassificationPipeline model(std::move(module), preprocessing(), 5, "", + {}); + const std::vector pixels(12, 0.75F); + require(model.run(request(pixels), {}).scores == expected, + "a single named classifier output retains the existing runtime contract"); +} + +} // namespace + +int main() { + try { + test_binding_and_complete_owned_logits(); + test_metadata_and_invalid_inputs(); + test_existing_named_output(); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_xception/tests/manifests/xception41-tf-in1k.json b/families/timm_xception/tests/manifests/xception41-tf-in1k.json index 132901f93d..d1d814865a 100644 --- a/families/timm_xception/tests/manifests/xception41-tf-in1k.json +++ b/families/timm_xception/tests/manifests/xception41-tf-in1k.json @@ -4,7 +4,7 @@ "hf_revision": "8a17189361e63c972815ef62f2a30dd5b9f393b1", "bundle": "xception41-tf-in1k.bundle", "family": "timm_xception", - "task": "classification", + "task": "image_to_class_scores", "precision": "fp16", "testcases": [ { diff --git a/families/timm_xception/tests/performance.yaml b/families/timm_xception/tests/performance.yaml new file mode 100644 index 0000000000..601b0ce000 --- /dev/null +++ b/families/timm_xception/tests/performance.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.perf-suite/v2 +name: timm-xception-performance +entries: + - id: timm_xception.classify + family: timm_xception + operation: classify + model: xception41-tf-in1k + workload: + testcase: xception41-tf-in1k + measurement: + warmup: 3 + iterations: 10 + equivalence_margin_percent: 5.0 + baseline: + runner: task-reference + script: tests/performance_reference.py + mode: hf-eager + reference_backend: hf_transformers + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false diff --git a/families/timm_xception/tests/performance_reference.py b/families/timm_xception/tests/performance_reference.py new file mode 100644 index 0000000000..7a4b2504ff --- /dev/null +++ b/families/timm_xception/tests/performance_reference.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Time Xception inference and host logits, not classification/reporting.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import statistics +import time + + +TIMING = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, +} + + +def _load_reference(arguments, request): + import timm + import torch + from huggingface_hub import snapshot_download + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + checkpoint = Path(arguments.model) + if not checkpoint.is_dir(): + checkpoint = Path(snapshot_download( + repo_id=arguments.model, revision=arguments.revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=arguments.local_files_only, + )) + config = json.loads((checkpoint / "config.json").read_text(encoding="utf-8")) + model = timm.create_model( + config["architecture"], pretrained=False, + pretrained_cfg=config["pretrained_cfg"], num_classes=int(config["num_classes"]), + checkpoint_path=str(checkpoint / "model.safetensors"), + ) + dtype = {"fp16": torch.float16, "fp32": torch.float32, "bf16": torch.bfloat16}[arguments.precision] + model = model.eval().to(device="cuda", dtype=dtype) + image_path = Path(request["image_path"]) + if not image_path.is_absolute(): + image_path = arguments.manifest.resolve().parent.parent / image_path + with Image.open(image_path) as image: + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image.convert("RGB")).unsqueeze(0).to(device="cuda", dtype=dtype) + + def invoke(): + with torch.inference_mode(): + # The SDK returns complete, host-owned float32 scores synchronously. + return model(inputs).to(device="cpu", dtype=torch.float32).numpy() + + return invoke, torch.cuda.synchronize, f"timm-{timm.__version__}" + + +def _measure(invoke, synchronize, warmup, iterations): + for _ in range(warmup): + invoke() + synchronize() + samples = [] + output = None + for _ in range(iterations): + output = None # As in the native worker, exclude prior-result destruction. + synchronize() + started = time.perf_counter() + output = invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + return samples, output + + +def _summary(scores): + import numpy as np + + if scores.ndim != 2 or scores.shape[0] != 1 or scores.shape[1] == 0: + raise ValueError("Xception must return one nonempty class-score vector") + if not np.isfinite(scores).all(): + raise ValueError("Xception returned nonfinite class scores") + return { + "top_class": int(scores[0].argmax()), "shape": list(scores.shape), + "element_count": int(scores.size), "finite": True, + "scores": scores[0].tolist(), + } + + +def run(arguments): + """Execute the existing family-reference protocol with reporting untimed.""" + if (arguments.family != "timm_xception" or arguments.operation != "classify" + or arguments.selected_task != "image_to_class_scores"): + raise ValueError("reference requires timm_xception image_to_class_scores/classify") + if arguments.warmup < 0 or arguments.iterations < 1: + raise ValueError("warmup must be nonnegative and iterations must be positive") + timing = json.loads(arguments.timing_contract_json) + if timing != TIMING or any(type(timing[key]) is not type(value) for key, value in TIMING.items()): + raise ValueError("reference requires its declared model-call timing policy") + if json.loads(arguments.adapter_options_json) or arguments.trust_remote_code: + raise ValueError("reference does not accept adapter options or remote code") + request = json.loads(arguments.request_json) + if (not isinstance(request, dict) or set(request) - {"image_path", "batch_size"} + or not isinstance(request.get("image_path"), str) or not request["image_path"] + or type(request.get("batch_size", 1)) is not int or request.get("batch_size", 1) != 1): + raise ValueError("reference requires one image_path and no runtime Config") + invoke, synchronize, framework = _load_reference(arguments, request) + samples, scores = _measure(invoke, synchronize, arguments.warmup, arguments.iterations) + return { + "schema_version": "trtmc.perf-baseline/v1", "status": "completed", + "model": arguments.model, "family": arguments.family, "operation": arguments.operation, + "case_name": arguments.case_name, "selected_task": arguments.selected_task, + "precision": arguments.precision, "mode": arguments.mode, "framework": framework, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": timing, **timing, "samples_ms": samples, + "metrics": {"latency_ms": {"p50": statistics.median(samples)}}, + "output_summary": _summary(scores), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model", "family", "operation", "selected-task", "request-json", + "adapter-options-json", "timing-contract-json", "case-name"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), required=True) + parser.add_argument("--mode", choices=("hf-eager",), required=True) + parser.add_argument("--padding", choices=("longest",), default="longest") + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--iterations", type=int, required=True) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + payload = run(arguments) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(payload, allow_nan=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/families/timm_xception/tests/sdk_consumer.c b/families/timm_xception/tests/sdk_consumer.c new file mode 100644 index 0000000000..2a88b5dccc --- /dev/null +++ b/families/timm_xception/tests/sdk_consumer.c @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +_Static_assert(sizeof(float) == 4, "input uses float32"); + +static trtmc_string_view text(const char* value) { + const trtmc_string_view result = {value, (uint64_t)strlen(value)}; + return result; +} +static uint32_t dimension(const char* text_value) { + char* end = NULL; + uintmax_t value; + errno = 0; + value = strtoumax(text_value, &end, 10); + if (errno || end == text_value || *end || !value || value > INT32_MAX) + return 0; + return (uint32_t)value; +} +static void json_string(trtmc_string_view value) { + uint64_t i; + putchar('"'); + for (i = 0; i < value.size; ++i) { + const unsigned char byte = (unsigned char)value.data[i]; + if (byte == '"' || byte == '\\') { + putchar('\\'); + putchar(byte); + } else if (byte < 32) { + printf("\\u%04x", (unsigned)byte); + } else { + putchar(byte); + } + } + putchar('"'); +} + +int main(int argc, char** argv) { + const trtmc_core_api_v1* core = NULL; + const trtmc_api_header* header = NULL; + const trtmc_image_to_class_scores_api_v1* task = NULL; + trtmc_model* model = NULL; + trtmc_result* result = NULL; + trtmc_error* error = NULL; + trtmc_load_options_v1 options = {0}; + trtmc_image_to_class_scores_request_v1 request = {0}; + trtmc_label_scores_view_v1 view = {0}; + float* input = NULL; + FILE* file = NULL; + uint32_t height, width; + size_t count; + uint64_t i, top_class = 0, field_count = 0; + int status = 1; + if (argc != 6) { + fprintf(stderr, "Usage: %s BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n", argv[0]); + return 2; + } + height = dimension(argv[4]); + width = dimension(argv[5]); + if (!height || !width || (size_t)height > SIZE_MAX / width / 3 / sizeof(float)) { + fprintf(stderr, "invalid or overflowing image dimensions\n"); + return 2; + } + count = (size_t)height * width * 3; + input = (float*)malloc(count * sizeof(float)); + file = fopen(argv[3], "rb"); + if (!input || !file || fread(input, sizeof(float), count, file) != count || + fgetc(file) != EOF || ferror(file)) { + fprintf(stderr, "input must contain exactly HEIGHT x WIDTH x 3 float32 RGB values\n"); + goto cleanup; + } + fclose(file); + file = NULL; + if (trtmc_get_api(1, 0, &core) != TRTMC_OK || !core) { + fprintf(stderr, "unable to obtain the v1 C API\n"); + goto cleanup; + } + options.struct_size = sizeof(options); + options.runtime_root = text(argv[2]); + if (core->model_load(text(argv[1]), &options, &model, &error) != TRTMC_OK) + goto cleanup; + if (core->model_get_task_api(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &header, + &error) != TRTMC_OK) + goto cleanup; + if (!header || header->byte_size < sizeof(*task)) { + fprintf(stderr, "incomplete image classification C table\n"); + goto cleanup; + } + task = (const trtmc_image_to_class_scores_api_v1*)header; + if (core->config_field_count(model, text(TRTMC_TASK_IMAGE_TO_CLASS_SCORES), 1, 0, &field_count, + &error) != TRTMC_OK) + goto cleanup; + if (field_count != 0) { + fprintf(stderr, "Xception must expose no runtime Config fields\n"); + goto cleanup; + } + request.image = + (trtmc_image_input_v1){input, count * sizeof(float), height, width, 3, TRTMC_IMAGE_FLOAT32}; + if (task->run(model, &request, NULL, &result, &error) != TRTMC_OK) + goto cleanup; + free(input); + input = NULL; + core->model_release(model); + model = NULL; + if (task->result_view(result, &view, &error) != TRTMC_OK) + goto cleanup; + if (!view.count || view.kind != TRTMC_SCORE_LOGIT) { + fprintf(stderr, "Xception must provide complete, unnormalized logits\n"); + goto cleanup; + } + for (i = 0; i < view.count; ++i) { + if (!isfinite(view.scores[i])) { + fprintf(stderr, "classification contains nonfinite logits\n"); + goto cleanup; + } + if (view.scores[i] > view.scores[top_class]) + top_class = i; + } + printf("{\"task\":\"image_to_class_scores\",\"input_shape\":[%" PRIu32 ",%" PRIu32 + ",3],\"kind\":\"logit\",\"score_kind\":%" PRIu32 ",\"vocabulary_id\":", + height, width, view.kind); + json_string(view.vocabulary_id); + printf(",\"labels\":["); + for (i = 0; i < view.labels.size; ++i) { + if (i) + putchar(','); + json_string(view.labels.data[i]); + } + printf("],\"top_class\":%" PRIu64 ",\"top_score\":%.*g,\"score_count\":%" PRIu64 + ",\"scores\":[", + top_class, FLT_DECIMAL_DIG, (double)view.scores[top_class], view.count); + for (i = 0; i < view.count; ++i) { + if (i) + putchar(','); + printf("%.*g", FLT_DECIMAL_DIG, (double)view.scores[i]); + } + puts("]}"); + status = fflush(stdout) == 0 && !ferror(stdout) ? 0 : 1; +cleanup: + if (error && core) { + const trtmc_string_view message = core->error_message(error); + fprintf(stderr, "C API error %d: ", (int)core->error_code(error)); + fwrite(message.data, 1, (size_t)message.size, stderr); + fputc('\n', stderr); + core->error_release(error); + } + if (core) { + core->result_release(result); + core->model_release(model); + } + if (file) + fclose(file); + free(input); + return status; +} diff --git a/families/timm_xception/tests/sdk_consumer.cpp b/families/timm_xception/tests/sdk_consumer.cpp new file mode 100644 index 0000000000..833ae65dca --- /dev/null +++ b/families/timm_xception/tests/sdk_consumer.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Public SDK consumer for the family-owned end-to-end test. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::uint32_t dimension(const char* argument) { + const std::string text(argument); + std::size_t consumed = 0; + const auto value = std::stoull(text, &consumed); + if (consumed != text.size() || !value || value > std::numeric_limits::max()) + throw std::invalid_argument("image dimensions must be positive int32 values"); + return static_cast(value); +} +void json_string(std::string_view value) { + std::cout << '"'; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') + std::cout << '\\' << static_cast(byte); + else if (byte < 32) + std::cout << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(byte) << std::dec << std::setfill(' '); + else + std::cout << static_cast(byte); + } + std::cout << '"'; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] << " BUNDLE RUNTIME_ROOT RGB_F32 HEIGHT WIDTH\n"; + return 2; + } + try { + static_assert(sizeof(float) == 4, "input uses float32"); + const auto height = dimension(argv[4]), width = dimension(argv[5]); + const auto limit = static_cast(std::numeric_limits::max()); + if (height > limit / width / 3 / sizeof(float)) + throw std::invalid_argument("image byte count overflows input storage"); + const auto count = static_cast(height) * width * 3; + std::vector input(count); + std::ifstream file(argv[3], std::ios::binary); + file.read(reinterpret_cast(input.data()), + static_cast(count * sizeof(float))); + if (!file || file.peek() != std::char_traits::eof()) + throw std::runtime_error( + "input must contain exactly HEIGHT x WIDTH x 3 RGB float32 values"); + auto result = [&] { + trtmc::LoadOptions options; + options.runtime_root = argv[2]; + auto model = trtmc::Model::load(argv[1], options); + const auto task = model.task(); + if (!task.config_fields().empty()) + throw std::runtime_error("Xception must expose no runtime Config fields"); + return task.run({trtmc::ImageInput({input.data(), input.size()}, height, width)}); + }(); + input.clear(); + input.shrink_to_fit(); + const auto scores = result.scores(); + if (scores.empty() || result.kind() != TRTMC_SCORE_LOGIT) + throw std::runtime_error("Xception must provide complete, unnormalized logits"); + std::size_t top = 0; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (!std::isfinite(scores[i])) + throw std::runtime_error("classification contains nonfinite logits"); + if (scores[i] > scores[top]) + top = i; + } + std::cout << std::setprecision(std::numeric_limits::max_digits10) + << "{\"task\":\"image_to_class_scores\",\"input_shape\":[" << height << ',' + << width << ",3],\"kind\":\"logit\",\"score_kind\":" << result.kind() + << ",\"vocabulary_id\":"; + json_string(result.vocabulary_id()); + std::cout << ",\"labels\":["; + const auto labels = result.labels(); + for (std::size_t i = 0; i < labels.size(); ++i) { + if (i) + std::cout << ','; + json_string(labels[i]); + } + std::cout << "],\"top_class\":" << top << ",\"top_score\":" << scores[top] + << ",\"score_count\":" << scores.size() << ",\"scores\":["; + for (std::size_t i = 0; i < scores.size(); ++i) { + if (i) + std::cout << ','; + std::cout << scores[i]; + } + std::cout << "]}\n"; + std::cout.flush(); + if (!std::cout) + throw std::runtime_error("failed to write classification output"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/timm_xception/tests/test_e2e.py b/families/timm_xception/tests/test_e2e.py index 91e2da600d..b6c3c2976f 100644 --- a/families/timm_xception/tests/test_e2e.py +++ b/families/timm_xception/tests/test_e2e.py @@ -8,6 +8,7 @@ from tools.e2e_evidence import evidence_stage, record_evidence import json +import math import os import subprocess from pathlib import Path @@ -28,7 +29,7 @@ def _cases() -> dict[str, tuple[dict, dict]]: for path in sorted(MANIFEST_ROOT.glob("*.json")): manifest = json.loads(path.read_text(encoding="utf-8")) assert manifest["family"] == FAMILY - assert manifest["task"] == "classification" + assert manifest["task"] == "image_to_class_scores" for case in manifest["testcases"]: name = str(case["name"]) assert name not in result @@ -105,6 +106,58 @@ def _asset(case: dict) -> Path: return path +def _assert_sdk_consumers( + runtime_root: Path, bundle: Path, case: dict, model_config: dict, expected: int, tmp_path: Path +) -> None: + import numpy as np + from PIL import Image + + native_build = _required_path(os.environ.get("TRTMC_NATIVE_BUILD_DIR"), "TRTMC_NATIVE_BUILD_DIR") + image = np.asarray(Image.open(_asset(case)).convert("RGB"), dtype=np.float32) + image /= np.float32(255.0) + raw_image = tmp_path / "sdk-input.rgb.f32" + image.tofile(raw_image) + outputs = [] + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ":".join( + value for value in (str(runtime_root), env.get("LD_LIBRARY_PATH", "")) if value + ) + for language in ("c", "cpp"): + consumer = native_build / f"test_timm_xception_sdk_{language}" + assert consumer.is_file(), f"build the family-owned SDK consumer: {consumer.name}" + completed = subprocess.run( + [ + str(consumer), + str(bundle), + str(runtime_root), + str(raw_image), + str(image.shape[0]), + str(image.shape[1]), + ], + check=True, + capture_output=True, + text=True, + env=env, + timeout=int(case.get("runtime_timeout_s", 3600)), + ) + payloads = [json.loads(line) for line in completed.stdout.splitlines() if line.startswith("{")] + assert len(payloads) == 1, f"SDK {language} consumer must return one result" + actual = payloads[0] + record_evidence(f"sdk_{language}", actual) + assert actual["task"] == "image_to_class_scores" + assert actual["kind"] == "logit" and actual["score_kind"] == 1 + assert actual["score_count"] == len(actual["scores"]) == int(model_config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_class"] == expected + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == model_config.get("vocabulary_id", "") + assert actual["labels"] == model_config.get("label_names", []) + outputs.append(actual) + # Both SDKs receive identical pixels. The original CLI test above keeps + # its JPEG decoder and the unchanged top-1 timm oracle. + assert outputs[0] == outputs[1] + + def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: manifest, case = CASES[case_name] record_evidence("inputs", {"manifest": manifest, "case": CASES[case_name][-1]}) @@ -177,3 +230,12 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: ) with evidence_stage("compare"): assert int(actual["top_class"]) == int(np.argmax(expected)) + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + assert actual["score_kind"] == "logit" and actual["scores"] == actual["logits"] + assert len(actual["scores"]) == int(config["num_classes"]) + assert all(math.isfinite(value) for value in actual["scores"]) + assert actual["top_score"] == max(actual["scores"]) + assert actual["vocabulary_id"] == config.get("vocabulary_id", "") + assert actual["labels"] == config.get("label_names", []) + with evidence_stage("sdk"): + _assert_sdk_consumers(runtime_root, bundle, case, config, int(np.argmax(expected)), tmp_path) diff --git a/families/timm_xception/tests/test_performance_reference.py b/families/timm_xception/tests/test_performance_reference.py new file mode 100644 index 0000000000..f736f81fba --- /dev/null +++ b/families/timm_xception/tests/test_performance_reference.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the family's benchmark timing and reference receipt.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from families.timm_xception.tests import performance_reference as reference + + +def arguments(**updates): + values = { + "family": "timm_xception", "operation": "classify", + "selected_task": "image_to_class_scores", "model": "fixture/model", + "case_name": "timm_xception.classify", "precision": "fp16", "mode": "hf-eager", + "warmup": 1, "iterations": 2, "timing_contract_json": json.dumps(reference.TIMING), + "adapter_options_json": "{}", "trust_remote_code": False, + "request_json": '{"image_path":"image.jpeg"}', + } + return SimpleNamespace(**(values | updates)) + + +def test_timing_stops_before_classification_and_receipt(monkeypatch): + events = [] + clocks = iter((1.0, 1.01, 2.0, 2.02)) + scores = np.array([[1.0, 4.0, -2.0]], dtype=np.float32) + + def clock(): + events.append("clock") + return next(clocks) + + def invoke(): + events.append("model-and-host-scores") + return scores.copy() + + def synchronize(): + events.append("synchronize") + + summarize = reference._summary + + def summary(value): + events.append("argmax-finite-summary") + return summarize(value) + + monkeypatch.setattr(reference.time, "perf_counter", clock) + monkeypatch.setattr(reference, "_load_reference", lambda *_: (invoke, synchronize, "fixture")) + monkeypatch.setattr(reference, "_summary", summary) + result = json.loads(json.dumps(reference.run(arguments()), allow_nan=False)) + assert events == ["model-and-host-scores", "synchronize"] + 2 * [ + "synchronize", "clock", "model-and-host-scores", "synchronize", "clock", + ] + ["argmax-finite-summary"] + assert result["schema_version"] == "trtmc.perf-baseline/v1" + assert result["status"] == "completed" + for field in ("family", "operation", "selected_task", "model", "case_name", "precision", "mode"): + assert result[field] == getattr(arguments(), field) + assert result["measurement"] == {"warmup": 1, "iterations": 2} + assert result["measurement_policy"] == reference.TIMING + assert all(result[key] == value for key, value in reference.TIMING.items()) + assert result["samples_ms"] == pytest.approx([10.0, 20.0]) + assert result["metrics"]["latency_ms"]["p50"] == pytest.approx(15.0) + assert result["output_summary"] == { + "top_class": 1, "shape": [1, 3], "element_count": 3, "finite": True, + "scores": [1.0, 4.0, -2.0], + } + + +@pytest.mark.parametrize("scores", [[], [[]], [[1.0], [2.0]], [[float("nan")]], [[float("inf")]]]) +def test_invalid_scores_do_not_produce_a_success_receipt(scores, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: ( + lambda: np.asarray(scores, dtype=np.float32), lambda: None, "fixture", + )) + with pytest.raises(ValueError, match="class-score|class scores"): + reference.run(arguments(warmup=0, iterations=1)) + + +@pytest.mark.parametrize("updates", [ + {"selected_task": "classification"}, {"family": "other"}, {"iterations": 0}, + {"warmup": -1}, {"request_json": '{"image_path":"image.jpeg","top_k":1}'}, + {"request_json": '{"image_path":"image.jpeg","batch_size":2}'}, + {"adapter_options_json": '{"extra":true}'}, {"trust_remote_code": True}, + {"timing_contract_json": json.dumps(reference.TIMING | {"input_preparation_included": True})}, +]) +def test_invalid_contract_fails_before_model_loading(updates, monkeypatch): + monkeypatch.setattr(reference, "_load_reference", lambda *_: pytest.fail("unexpected model load")) + with pytest.raises(ValueError): + reference.run(arguments(**updates)) diff --git a/families/timm_xception/tests/test_sdk_metadata.py b/families/timm_xception/tests/test_sdk_metadata.py new file mode 100644 index 0000000000..832cc30fed --- /dev/null +++ b/families/timm_xception/tests/test_sdk_metadata.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The builder publishes complete checkpoint-owned Task metadata.""" + +from types import SimpleNamespace + +import pytest + +from families.timm_xception import model +from tensorrt_model_connect import BuildRequest + + +@pytest.mark.parametrize("metadata,invalid", [ + ({}, False), + ({"vocabulary_id": "fixture:five", "label_names": ["a", "b", "c", "d", "e"]}, False), + ({"label_names": ["only-one"]}, True), + ({"label_names": ["a", "b", "", "d", "e"]}, True), + ({"label_names": 5}, True), + ({"vocabulary_id": 5}, True), +]) +def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): + raw = {"num_classes": 5, **metadata} + runtime = {"image_height": 2, "image_width": 2, "num_classes": 5, + "crop_pct": 1.0, "interpolation": "bilinear", + "mean": [0.5] * 3, "std": [0.25] * 3} + monkeypatch.setattr(model, "_read_config", lambda _: raw) + monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) + monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + sections = {} + headers = [] + writer = SimpleNamespace(set_header=lambda **value: headers.append(value), + add_bytes=lambda key, value: sections.update({key: value}), + add_json=lambda key, value: sections.update({key: value})) + request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", + family="timm_xception", task="image_to_class_scores", precision="fp32") + if invalid: + with pytest.raises(ValueError, match="vocabulary_id|label_names"): + model.build(request, writer) + assert not sections and not headers + return + model.build(request, writer) + assert headers == [{"family": "timm_xception", "task": "image_to_class_scores", "backend": "trt"}] + assert sections["engine.plan"] == b"plan" + assert sections["runtime.json"]["num_classes"] == 5 + assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") + assert sections["runtime.json"]["labels"] == metadata.get("label_names", []) From 6b77b18fbe1924b36fd6d4d7138fd4bd6cb91478 Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:20:16 -0700 Subject: [PATCH 2/4] fix(vision): reject unnamed class metadata Validate supplied class names when vocabulary identity is unknown before executing a model. Preserve absent labels and explicit vocabulary identities, with family-owned positive and negative constructor regressions. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- families/timm_dpn/runtime/pipeline.cpp | 6 ++++ .../timm_dpn/tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ families/timm_hrnet/runtime/pipeline.cpp | 6 ++++ .../tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ families/timm_mnasnet/runtime/pipeline.cpp | 6 ++++ .../tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ families/timm_nfnet/runtime/pipeline.cpp | 6 ++++ .../tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ families/timm_repvgg/runtime/pipeline.cpp | 6 ++++ .../tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ families/timm_res2net/runtime/pipeline.cpp | 6 ++++ .../tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ families/timm_senet/runtime/pipeline.cpp | 6 ++++ .../tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ families/timm_seresnet/runtime/pipeline.cpp | 6 ++++ .../tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ families/timm_vgg/runtime/pipeline.cpp | 6 ++++ .../timm_vgg/tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ families/timm_xception/runtime/pipeline.cpp | 6 ++++ .../tests/cpp/test_task_contract.cpp | 29 +++++++++++++++++++ 20 files changed, 350 insertions(+) diff --git a/families/timm_dpn/runtime/pipeline.cpp b/families/timm_dpn/runtime/pipeline.cpp index 74894a2c0b..bab6a2ada9 100644 --- a/families/timm_dpn/runtime/pipeline.cpp +++ b/families/timm_dpn/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_dpn/runtime/pipeline.h" +#include #include #include #include @@ -38,6 +39,11 @@ TimmDpnImageClassificationPipeline::TimmDpnImageClassificationPipeline( if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm DPN class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_dpn/tests/cpp/test_task_contract.cpp b/families/timm_dpn/tests/cpp/test_task_contract.cpp index 1d761aa9d0..808038df17 100644 --- a/families/timm_dpn/tests/cpp/test_task_contract.cpp +++ b/families/timm_dpn/tests/cpp/test_task_contract.cpp @@ -118,6 +118,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmDpnImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmDpnImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -165,6 +193,7 @@ void test_existing_named_output() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); test_existing_named_output(); return 0; diff --git a/families/timm_hrnet/runtime/pipeline.cpp b/families/timm_hrnet/runtime/pipeline.cpp index 6a02e24412..98167b9501 100644 --- a/families/timm_hrnet/runtime/pipeline.cpp +++ b/families/timm_hrnet/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_hrnet/runtime/pipeline.h" +#include #include #include #include @@ -38,6 +39,11 @@ TimmHRNetImageClassificationPipeline::TimmHRNetImageClassificationPipeline( if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm HRNet class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_hrnet/tests/cpp/test_task_contract.cpp b/families/timm_hrnet/tests/cpp/test_task_contract.cpp index 812ed5d85a..95c3ed6a3b 100644 --- a/families/timm_hrnet/tests/cpp/test_task_contract.cpp +++ b/families/timm_hrnet/tests/cpp/test_task_contract.cpp @@ -118,6 +118,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmHRNetImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmHRNetImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -166,6 +194,7 @@ void test_existing_named_output() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); test_existing_named_output(); return 0; diff --git a/families/timm_mnasnet/runtime/pipeline.cpp b/families/timm_mnasnet/runtime/pipeline.cpp index 4dc52731c3..515d1c4a1b 100644 --- a/families/timm_mnasnet/runtime/pipeline.cpp +++ b/families/timm_mnasnet/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_mnasnet/runtime/pipeline.h" +#include #include #include #include @@ -37,6 +38,11 @@ TimmMnasnetImageClassificationPipeline::TimmMnasnetImageClassificationPipeline( if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm MNASNet class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_mnasnet/tests/cpp/test_task_contract.cpp b/families/timm_mnasnet/tests/cpp/test_task_contract.cpp index 8fecd371e8..4621ab9f6e 100644 --- a/families/timm_mnasnet/tests/cpp/test_task_contract.cpp +++ b/families/timm_mnasnet/tests/cpp/test_task_contract.cpp @@ -117,6 +117,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmMnasnetImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmMnasnetImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -154,6 +182,7 @@ void test_metadata_and_invalid_inputs() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); return 0; } catch (const std::exception& error) { diff --git a/families/timm_nfnet/runtime/pipeline.cpp b/families/timm_nfnet/runtime/pipeline.cpp index af0692d850..40774537a5 100644 --- a/families/timm_nfnet/runtime/pipeline.cpp +++ b/families/timm_nfnet/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_nfnet/runtime/pipeline.h" +#include #include #include #include @@ -38,6 +39,11 @@ TimmNfnetImageClassificationPipeline::TimmNfnetImageClassificationPipeline( if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm NFNet class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_nfnet/tests/cpp/test_task_contract.cpp b/families/timm_nfnet/tests/cpp/test_task_contract.cpp index 03af896965..ba874f5e40 100644 --- a/families/timm_nfnet/tests/cpp/test_task_contract.cpp +++ b/families/timm_nfnet/tests/cpp/test_task_contract.cpp @@ -118,6 +118,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmNfnetImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmNfnetImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -166,6 +194,7 @@ void test_existing_named_output() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); test_existing_named_output(); return 0; diff --git a/families/timm_repvgg/runtime/pipeline.cpp b/families/timm_repvgg/runtime/pipeline.cpp index 002756ccfb..1f2757ea28 100644 --- a/families/timm_repvgg/runtime/pipeline.cpp +++ b/families/timm_repvgg/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_repvgg/runtime/pipeline.h" +#include #include #include #include @@ -38,6 +39,11 @@ TimmRepvggImageClassificationPipeline::TimmRepvggImageClassificationPipeline( if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm RepVGG class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_repvgg/tests/cpp/test_task_contract.cpp b/families/timm_repvgg/tests/cpp/test_task_contract.cpp index 410cf566a7..9567117118 100644 --- a/families/timm_repvgg/tests/cpp/test_task_contract.cpp +++ b/families/timm_repvgg/tests/cpp/test_task_contract.cpp @@ -118,6 +118,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmRepvggImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmRepvggImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -166,6 +194,7 @@ void test_existing_named_output() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); test_existing_named_output(); return 0; diff --git a/families/timm_res2net/runtime/pipeline.cpp b/families/timm_res2net/runtime/pipeline.cpp index 1a74bf35fc..75e065fd1e 100644 --- a/families/timm_res2net/runtime/pipeline.cpp +++ b/families/timm_res2net/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_res2net/runtime/pipeline.h" +#include #include #include #include @@ -38,6 +39,11 @@ TimmRes2NetImageClassificationPipeline::TimmRes2NetImageClassificationPipeline( if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm Res2Net class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_res2net/tests/cpp/test_task_contract.cpp b/families/timm_res2net/tests/cpp/test_task_contract.cpp index b7bfdbb646..d36e7af543 100644 --- a/families/timm_res2net/tests/cpp/test_task_contract.cpp +++ b/families/timm_res2net/tests/cpp/test_task_contract.cpp @@ -118,6 +118,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmRes2NetImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmRes2NetImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -166,6 +194,7 @@ void test_existing_named_output() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); test_existing_named_output(); return 0; diff --git a/families/timm_senet/runtime/pipeline.cpp b/families/timm_senet/runtime/pipeline.cpp index 12524781fe..1f1d21210a 100644 --- a/families/timm_senet/runtime/pipeline.cpp +++ b/families/timm_senet/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_senet/runtime/pipeline.h" +#include #include #include #include @@ -38,6 +39,11 @@ TimmSENetImageClassificationPipeline::TimmSENetImageClassificationPipeline( if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm SENet class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_senet/tests/cpp/test_task_contract.cpp b/families/timm_senet/tests/cpp/test_task_contract.cpp index 0aebe10ac5..cd44f4c9cb 100644 --- a/families/timm_senet/tests/cpp/test_task_contract.cpp +++ b/families/timm_senet/tests/cpp/test_task_contract.cpp @@ -118,6 +118,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmSENetImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmSENetImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -166,6 +194,7 @@ void test_existing_named_output() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); test_existing_named_output(); return 0; diff --git a/families/timm_seresnet/runtime/pipeline.cpp b/families/timm_seresnet/runtime/pipeline.cpp index a015cb0705..0a50d0333d 100644 --- a/families/timm_seresnet/runtime/pipeline.cpp +++ b/families/timm_seresnet/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_seresnet/runtime/pipeline.h" +#include #include #include #include @@ -38,6 +39,11 @@ TimmSEResNetImageClassificationPipeline::TimmSEResNetImageClassificationPipeline if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm SE-ResNet class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_seresnet/tests/cpp/test_task_contract.cpp b/families/timm_seresnet/tests/cpp/test_task_contract.cpp index 4683c2324d..4d67075b55 100644 --- a/families/timm_seresnet/tests/cpp/test_task_contract.cpp +++ b/families/timm_seresnet/tests/cpp/test_task_contract.cpp @@ -118,6 +118,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmSEResNetImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmSEResNetImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -166,6 +194,7 @@ void test_existing_named_output() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); test_existing_named_output(); return 0; diff --git a/families/timm_vgg/runtime/pipeline.cpp b/families/timm_vgg/runtime/pipeline.cpp index e13a79e77e..4cd2c0c693 100644 --- a/families/timm_vgg/runtime/pipeline.cpp +++ b/families/timm_vgg/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_vgg/runtime/pipeline.h" +#include #include #include #include @@ -37,6 +38,11 @@ TimmVggImageClassificationPipeline::TimmVggImageClassificationPipeline( if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm VGG class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_vgg/tests/cpp/test_task_contract.cpp b/families/timm_vgg/tests/cpp/test_task_contract.cpp index 3d2f21aac2..861c9eda69 100644 --- a/families/timm_vgg/tests/cpp/test_task_contract.cpp +++ b/families/timm_vgg/tests/cpp/test_task_contract.cpp @@ -117,6 +117,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmVggImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmVggImageClassificationPipeline model(std::make_unique(), + preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -154,6 +182,7 @@ void test_metadata_and_invalid_inputs() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); return 0; } catch (const std::exception& error) { diff --git a/families/timm_xception/runtime/pipeline.cpp b/families/timm_xception/runtime/pipeline.cpp index a6048203c2..3b2cca6901 100644 --- a/families/timm_xception/runtime/pipeline.cpp +++ b/families/timm_xception/runtime/pipeline.cpp @@ -5,6 +5,7 @@ #include "families/timm_xception/runtime/pipeline.h" +#include #include #include #include @@ -38,6 +39,11 @@ TimmXceptionImageClassificationPipeline::TimmXceptionImageClassificationPipeline if (num_classes_ <= 0 || (!labels_.empty() && labels_.size() != static_cast(num_classes_))) throw std::runtime_error("timm Xception class metadata does not match its output size"); + if (vocabulary_id_.empty() && + std::any_of(labels_.begin(), labels_.end(), + [](const std::string& label) { return label.empty(); })) + throw std::runtime_error( + "class labels require nonempty names without an explicit vocabulary identity"); } internal::LabelScoresResult diff --git a/families/timm_xception/tests/cpp/test_task_contract.cpp b/families/timm_xception/tests/cpp/test_task_contract.cpp index 0047fb64b3..874a9caa18 100644 --- a/families/timm_xception/tests/cpp/test_task_contract.cpp +++ b/families/timm_xception/tests/cpp/test_task_contract.cpp @@ -118,6 +118,34 @@ void test_binding_and_complete_owned_logits() { "result must own all logits across another call and model destruction"); } +void test_class_identity_metadata() { + const std::vector partial{"first", "", "third", "fourth", "fifth"}; + for (const auto& labels : {partial, std::vector(5)}) { + rejects( + [&] { + trtmc::TimmXceptionImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, "", labels); + }, + "reject blank class labels without vocabulary identity at construction"); + } + + const std::pair> valid[] = { + {"", {}}, + {"", {"first", "second", "third", "fourth", "fifth"}}, + {"test:five-classes", {}}, + {"test:five-classes", partial}, + {"test:five-classes", std::vector(5)}, + }; + const std::vector pixels(12, 0.75F); + for (const auto& [vocabulary, labels] : valid) { + trtmc::TimmXceptionImageClassificationPipeline model( + std::make_unique(), preprocessing(), 5, vocabulary, labels); + const auto result = model.run(request(pixels), {}); + require(result.vocabulary_id == vocabulary && result.labels == labels, + "preserve valid unknown, named and explicitly identified class metadata"); + } +} + void test_metadata_and_invalid_inputs() { auto module = std::make_unique(); auto* recording = module.get(); @@ -166,6 +194,7 @@ void test_existing_named_output() { int main() { try { test_binding_and_complete_owned_logits(); + test_class_identity_metadata(); test_metadata_and_invalid_inputs(); test_existing_named_output(); return 0; From a9221a6e27d0e59ac56e895a7aa41fada8442a8f Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:23:28 -0700 Subject: [PATCH 3/4] test(vision): cover dependency-free Task discovery Exercise each migrated family identity and default Task using the existing CPU-only support-test convention, including rejection of unrelated identities. Keep shared architecture gates unchanged. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- families/timm_dpn/tests/test_support.py | 18 ++++++++++++++++++ families/timm_hrnet/tests/test_support.py | 18 ++++++++++++++++++ families/timm_mnasnet/tests/test_support.py | 18 ++++++++++++++++++ families/timm_nfnet/tests/test_support.py | 18 ++++++++++++++++++ families/timm_repvgg/tests/test_support.py | 18 ++++++++++++++++++ families/timm_res2net/tests/test_support.py | 18 ++++++++++++++++++ families/timm_senet/tests/test_support.py | 18 ++++++++++++++++++ families/timm_seresnet/tests/test_support.py | 18 ++++++++++++++++++ families/timm_vgg/tests/test_support.py | 18 ++++++++++++++++++ families/timm_xception/tests/test_support.py | 18 ++++++++++++++++++ 10 files changed, 180 insertions(+) create mode 100644 families/timm_dpn/tests/test_support.py create mode 100644 families/timm_hrnet/tests/test_support.py create mode 100644 families/timm_mnasnet/tests/test_support.py create mode 100644 families/timm_nfnet/tests/test_support.py create mode 100644 families/timm_repvgg/tests/test_support.py create mode 100644 families/timm_res2net/tests/test_support.py create mode 100644 families/timm_senet/tests/test_support.py create mode 100644 families/timm_seresnet/tests/test_support.py create mode 100644 families/timm_vgg/tests/test_support.py create mode 100644 families/timm_xception/tests/test_support.py diff --git a/families/timm_dpn/tests/test_support.py b/families/timm_dpn/tests/test_support.py new file mode 100644 index 0000000000..bd7a22271b --- /dev/null +++ b/families/timm_dpn/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_dpn.""" + +from families.timm_dpn.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "dpn68b"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None diff --git a/families/timm_hrnet/tests/test_support.py b/families/timm_hrnet/tests/test_support.py new file mode 100644 index 0000000000..83607ba9d0 --- /dev/null +++ b/families/timm_hrnet/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_hrnet.""" + +from families.timm_hrnet.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "hrnet_w18"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None diff --git a/families/timm_mnasnet/tests/test_support.py b/families/timm_mnasnet/tests/test_support.py new file mode 100644 index 0000000000..be91231612 --- /dev/null +++ b/families/timm_mnasnet/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_mnasnet.""" + +from families.timm_mnasnet.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "mnasnet_100"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None diff --git a/families/timm_nfnet/tests/test_support.py b/families/timm_nfnet/tests/test_support.py new file mode 100644 index 0000000000..3a302a111b --- /dev/null +++ b/families/timm_nfnet/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_nfnet.""" + +from families.timm_nfnet.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "dm_nfnet_f0"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None diff --git a/families/timm_repvgg/tests/test_support.py b/families/timm_repvgg/tests/test_support.py new file mode 100644 index 0000000000..6c6586aa90 --- /dev/null +++ b/families/timm_repvgg/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_repvgg.""" + +from families.timm_repvgg.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "repvgg_a2"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None diff --git a/families/timm_res2net/tests/test_support.py b/families/timm_res2net/tests/test_support.py new file mode 100644 index 0000000000..dfec296bb6 --- /dev/null +++ b/families/timm_res2net/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_res2net.""" + +from families.timm_res2net.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "res2net50_26w_4s"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None diff --git a/families/timm_senet/tests/test_support.py b/families/timm_senet/tests/test_support.py new file mode 100644 index 0000000000..787123f0aa --- /dev/null +++ b/families/timm_senet/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_senet.""" + +from families.timm_senet.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "senet154"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None diff --git a/families/timm_seresnet/tests/test_support.py b/families/timm_seresnet/tests/test_support.py new file mode 100644 index 0000000000..d35e8b57be --- /dev/null +++ b/families/timm_seresnet/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_seresnet.""" + +from families.timm_seresnet.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "seresnet50"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None diff --git a/families/timm_vgg/tests/test_support.py b/families/timm_vgg/tests/test_support.py new file mode 100644 index 0000000000..0dccda4f1c --- /dev/null +++ b/families/timm_vgg/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_vgg.""" + +from families.timm_vgg.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "vgg16"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None diff --git a/families/timm_xception/tests/test_support.py b/families/timm_xception/tests/test_support.py new file mode 100644 index 0000000000..56801a3a29 --- /dev/null +++ b/families/timm_xception/tests/test_support.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dependency-free semantic Task discovery for timm_xception.""" + +from families.timm_xception.support import describe +from tensorrt_model_connect.model_support import ModelMetadata + + +def test_primary_task_matches_the_semantic_runtime(): + support = describe(ModelMetadata(config={"architecture": "xception41"}, model_index={})) + assert support is not None + assert support.tasks == ("image_to_class_scores",) + assert support.default_task == "image_to_class_scores" + + +def test_unrelated_identity_is_not_claimed(): + assert describe(ModelMetadata(config={"architecture": "unrelated-model"}, model_index={})) is None From 9d4a8b2658684a3f78cb6025bf1aefa774d2fa08 Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:43:57 -0700 Subject: [PATCH 4/4] fix(vision): validate metadata before engine builds Reuse existing family metadata checks before TensorRT engine construction. Preserve valid metadata and error rules, and verify invalid metadata never calls the engine builder. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- families/timm_dpn/model.py | 14 ++++++------ families/timm_dpn/tests/test_sdk_metadata.py | 10 ++++++++- families/timm_hrnet/model.py | 14 ++++++------ .../timm_hrnet/tests/test_sdk_metadata.py | 10 ++++++++- families/timm_mnasnet/model.py | 2 +- families/timm_mnasnet/tests/test_model.py | 22 ++++++++++++++----- families/timm_nfnet/model.py | 8 +++---- .../timm_nfnet/tests/test_sdk_metadata.py | 10 ++++++++- families/timm_repvgg/model.py | 14 ++++++------ .../timm_repvgg/tests/test_sdk_metadata.py | 10 ++++++++- families/timm_res2net/model.py | 14 ++++++------ .../timm_res2net/tests/test_sdk_metadata.py | 10 ++++++++- families/timm_senet/model.py | 14 ++++++------ .../timm_senet/tests/test_sdk_metadata.py | 10 ++++++++- families/timm_seresnet/model.py | 14 ++++++------ .../timm_seresnet/tests/test_sdk_metadata.py | 10 ++++++++- families/timm_vgg/model.py | 2 +- families/timm_vgg/tests/test_model.py | 22 ++++++++++++++----- families/timm_xception/model.py | 14 ++++++------ .../timm_xception/tests/test_sdk_metadata.py | 10 ++++++++- 20 files changed, 161 insertions(+), 73 deletions(-) diff --git a/families/timm_dpn/model.py b/families/timm_dpn/model.py index dc52d7c42a..ab34bbaa8b 100644 --- a/families/timm_dpn/model.py +++ b/families/timm_dpn/model.py @@ -468,21 +468,21 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_dpn supports only max_sequence_length=1") model_dir = Path(request.model_dir) raw = _read_config(model_dir) - plan, runtime = _build_engine( - raw, - Checkpoint.open(model_dir), - str(request.precision).lower(), - bool(request.verbose), - ) vocabulary_id = raw.get("vocabulary_id", "") labels = raw.get("label_names", []) if not isinstance(vocabulary_id, str): raise ValueError("timm DPN vocabulary_id must be a string") if not isinstance(labels, list) or (labels and ( - len(labels) != runtime["num_classes"] + len(labels) != _preprocess_config(raw)["num_classes"] or any(not isinstance(label, str) or not label for label in labels) )): raise ValueError("timm DPN label_names must name every class") + plan, runtime = _build_engine( + raw, + Checkpoint.open(model_dir), + str(request.precision).lower(), + bool(request.verbose), + ) writer.set_header(family="timm_dpn", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_dpn/tests/test_sdk_metadata.py b/families/timm_dpn/tests/test_sdk_metadata.py index 50c1c7fa3e..8e66c91e7c 100644 --- a/families/timm_dpn/tests/test_sdk_metadata.py +++ b/families/timm_dpn/tests/test_sdk_metadata.py @@ -26,7 +26,13 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): "mean": [0.5] * 3, "std": [0.25] * 3} monkeypatch.setattr(model, "_read_config", lambda _: raw) monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) - monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + engine_calls = [] + + def build_engine(*args): + engine_calls.append(args) + return b"plan", runtime + + monkeypatch.setattr(model, "_build_engine", build_engine) sections = {} headers = [] writer = SimpleNamespace(set_header=lambda **value: headers.append(value), @@ -38,8 +44,10 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): with pytest.raises(ValueError, match="vocabulary_id|label_names"): model.build(request, writer) assert not sections and not headers + assert not engine_calls return model.build(request, writer) + assert len(engine_calls) == 1 assert headers == [{"family": "timm_dpn", "task": "image_to_class_scores", "backend": "trt"}] assert sections["engine.plan"] == b"plan" assert sections["runtime.json"]["num_classes"] == 5 diff --git a/families/timm_hrnet/model.py b/families/timm_hrnet/model.py index ecd9645fe6..9fb12072cf 100644 --- a/families/timm_hrnet/model.py +++ b/families/timm_hrnet/model.py @@ -471,21 +471,21 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: _positive_int(request.max_sequence_length or 1, "max_sequence_length") model_dir = Path(request.model_dir) raw = _read_config(model_dir) - plan, runtime = _build_engine( - raw, - Checkpoint.open(model_dir), - str(request.precision).lower(), - bool(request.verbose), - ) vocabulary_id = raw.get("vocabulary_id", "") labels = raw.get("label_names", []) if not isinstance(vocabulary_id, str): raise ValueError("timm HRNet vocabulary_id must be a string") if not isinstance(labels, list) or (labels and ( - len(labels) != runtime["num_classes"] + len(labels) != _preprocess_config(raw)["num_classes"] or any(not isinstance(label, str) or not label for label in labels) )): raise ValueError("timm HRNet label_names must name every class") + plan, runtime = _build_engine( + raw, + Checkpoint.open(model_dir), + str(request.precision).lower(), + bool(request.verbose), + ) writer.set_header(family="timm_hrnet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_hrnet/tests/test_sdk_metadata.py b/families/timm_hrnet/tests/test_sdk_metadata.py index 32b10b7598..9a64ad6064 100644 --- a/families/timm_hrnet/tests/test_sdk_metadata.py +++ b/families/timm_hrnet/tests/test_sdk_metadata.py @@ -26,7 +26,13 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): "mean": [0.5] * 3, "std": [0.25] * 3} monkeypatch.setattr(model, "_read_config", lambda _: raw) monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) - monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + engine_calls = [] + + def build_engine(*args): + engine_calls.append(args) + return b"plan", runtime + + monkeypatch.setattr(model, "_build_engine", build_engine) sections = {} headers = [] writer = SimpleNamespace(set_header=lambda **value: headers.append(value), @@ -38,8 +44,10 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): with pytest.raises(ValueError, match="vocabulary_id|label_names"): model.build(request, writer) assert not sections and not headers + assert not engine_calls return model.build(request, writer) + assert len(engine_calls) == 1 assert headers == [{"family": "timm_hrnet", "task": "image_to_class_scores", "backend": "trt"}] assert sections["engine.plan"] == b"plan" assert sections["runtime.json"]["num_classes"] == 5 diff --git a/families/timm_mnasnet/model.py b/families/timm_mnasnet/model.py index 21366deda5..c11c4b9d1c 100644 --- a/families/timm_mnasnet/model.py +++ b/families/timm_mnasnet/model.py @@ -447,13 +447,13 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: precision = str(request.precision).lower() model = _TimmMnasnetModel() weights = model.load_weights(str(model_dir), config, precision=precision) + runtime_source = model.get_bundle_config_overrides(config) plan = model.build_engine( config, weights, precision=precision, verbose=bool(request.verbose), ) - runtime_source = model.get_bundle_config_overrides(config) writer.set_header(family="timm_mnasnet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_mnasnet/tests/test_model.py b/families/timm_mnasnet/tests/test_model.py index 2f181a72ea..566d558634 100644 --- a/families/timm_mnasnet/tests/test_model.py +++ b/families/timm_mnasnet/tests/test_model.py @@ -242,14 +242,26 @@ def add_json(self, name, value): assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") -@pytest.mark.parametrize("labels", [["only-one"], ["a", "b", "", "d", "e"], 5]) -def test_build_rejects_incomplete_class_labels(tmp_path, monkeypatch, labels): +@pytest.mark.parametrize("metadata,message", [ + ({"label_names": ["only-one"]}, "label_names must name every class"), + ({"label_names": ["a", "b", "", "d", "e"]}, "label_names must name every class"), + ({"label_names": 5}, "label_names must name every class"), + ({"vocabulary_id": 5}, "vocabulary_id must be a string"), +]) +def test_build_rejects_invalid_class_metadata_before_engine(tmp_path, monkeypatch, metadata, message): _write_tiny_mnasnet(tmp_path) raw = json.loads((tmp_path / "config.json").read_text()) - raw["label_names"] = labels + raw.update(metadata) (tmp_path / "config.json").write_text(json.dumps(raw)) - monkeypatch.setattr(_TimmMnasnetModel, "build_engine", lambda *args, **kwargs: b"plan") + engine_calls = [] + + def build_engine(*args, **kwargs): + engine_calls.append((args, kwargs)) + return b"plan" + + monkeypatch.setattr(_TimmMnasnetModel, "build_engine", build_engine) request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", family="timm_mnasnet", task="image_to_class_scores", precision="fp32") - with pytest.raises(ValueError, match="label_names must name every class"): + with pytest.raises(ValueError, match=message): build_family(request, object()) + assert not engine_calls diff --git a/families/timm_nfnet/model.py b/families/timm_nfnet/model.py index 1903d3acaf..c8202ad555 100644 --- a/families/timm_nfnet/model.py +++ b/families/timm_nfnet/model.py @@ -446,18 +446,18 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: raise NotImplementedError("timm_nfnet supports only max_sequence_length=1") model_dir = Path(request.model_dir) raw = _read_config(model_dir) - plan, runtime = _build_engine( - raw, Checkpoint.open(model_dir), str(request.precision).lower(), bool(request.verbose) - ) vocabulary_id = raw.get("vocabulary_id", "") labels = raw.get("label_names", []) if not isinstance(vocabulary_id, str): raise ValueError("timm NFNet vocabulary_id must be a string") if not isinstance(labels, list) or (labels and ( - len(labels) != runtime["num_classes"] + len(labels) != _preprocess_config(raw)["num_classes"] or any(not isinstance(label, str) or not label for label in labels) )): raise ValueError("timm NFNet label_names must name every class") + plan, runtime = _build_engine( + raw, Checkpoint.open(model_dir), str(request.precision).lower(), bool(request.verbose) + ) writer.set_header(family="timm_nfnet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_nfnet/tests/test_sdk_metadata.py b/families/timm_nfnet/tests/test_sdk_metadata.py index 99b39ddf01..841eb3702e 100644 --- a/families/timm_nfnet/tests/test_sdk_metadata.py +++ b/families/timm_nfnet/tests/test_sdk_metadata.py @@ -26,7 +26,13 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): "mean": [0.5] * 3, "std": [0.25] * 3} monkeypatch.setattr(model, "_read_config", lambda _: raw) monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) - monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + engine_calls = [] + + def build_engine(*args): + engine_calls.append(args) + return b"plan", runtime + + monkeypatch.setattr(model, "_build_engine", build_engine) sections = {} headers = [] writer = SimpleNamespace(set_header=lambda **value: headers.append(value), @@ -38,8 +44,10 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): with pytest.raises(ValueError, match="vocabulary_id|label_names"): model.build(request, writer) assert not sections and not headers + assert not engine_calls return model.build(request, writer) + assert len(engine_calls) == 1 assert headers == [{"family": "timm_nfnet", "task": "image_to_class_scores", "backend": "trt"}] assert sections["engine.plan"] == b"plan" assert sections["runtime.json"]["num_classes"] == 5 diff --git a/families/timm_repvgg/model.py b/families/timm_repvgg/model.py index 29868c10c2..8472c88c04 100644 --- a/families/timm_repvgg/model.py +++ b/families/timm_repvgg/model.py @@ -338,21 +338,21 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: _positive_int(request.max_sequence_length or 1, "max_sequence_length") model_dir = Path(request.model_dir) raw = _read_config(model_dir) - plan, runtime = _build_engine( - raw, - Checkpoint.open(model_dir), - str(request.precision).lower(), - bool(request.verbose), - ) vocabulary_id = raw.get("vocabulary_id", "") labels = raw.get("label_names", []) if not isinstance(vocabulary_id, str): raise ValueError("timm RepVGG vocabulary_id must be a string") if not isinstance(labels, list) or (labels and ( - len(labels) != runtime["num_classes"] + len(labels) != _preprocess_config(raw)["num_classes"] or any(not isinstance(label, str) or not label for label in labels) )): raise ValueError("timm RepVGG label_names must name every class") + plan, runtime = _build_engine( + raw, + Checkpoint.open(model_dir), + str(request.precision).lower(), + bool(request.verbose), + ) writer.set_header(family="timm_repvgg", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_repvgg/tests/test_sdk_metadata.py b/families/timm_repvgg/tests/test_sdk_metadata.py index 88ed9dd61b..3bd8d7c219 100644 --- a/families/timm_repvgg/tests/test_sdk_metadata.py +++ b/families/timm_repvgg/tests/test_sdk_metadata.py @@ -26,7 +26,13 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): "mean": [0.5] * 3, "std": [0.25] * 3} monkeypatch.setattr(model, "_read_config", lambda _: raw) monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) - monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + engine_calls = [] + + def build_engine(*args): + engine_calls.append(args) + return b"plan", runtime + + monkeypatch.setattr(model, "_build_engine", build_engine) sections = {} headers = [] writer = SimpleNamespace(set_header=lambda **value: headers.append(value), @@ -38,8 +44,10 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): with pytest.raises(ValueError, match="vocabulary_id|label_names"): model.build(request, writer) assert not sections and not headers + assert not engine_calls return model.build(request, writer) + assert len(engine_calls) == 1 assert headers == [{"family": "timm_repvgg", "task": "image_to_class_scores", "backend": "trt"}] assert sections["engine.plan"] == b"plan" assert sections["runtime.json"]["num_classes"] == 5 diff --git a/families/timm_res2net/model.py b/families/timm_res2net/model.py index 73418fb114..ca63b408cc 100644 --- a/families/timm_res2net/model.py +++ b/families/timm_res2net/model.py @@ -479,21 +479,21 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: _positive_int(request.max_sequence_length or 1, "max_sequence_length") model_dir = Path(request.model_dir) raw = _read_config(model_dir) - plan, runtime = _build_engine( - raw, - Checkpoint.open(model_dir), - str(request.precision).lower(), - bool(request.verbose), - ) vocabulary_id = raw.get("vocabulary_id", "") labels = raw.get("label_names", []) if not isinstance(vocabulary_id, str): raise ValueError("timm Res2Net vocabulary_id must be a string") if not isinstance(labels, list) or (labels and ( - len(labels) != runtime["num_classes"] + len(labels) != _preprocess_config(raw)["num_classes"] or any(not isinstance(label, str) or not label for label in labels) )): raise ValueError("timm Res2Net label_names must name every class") + plan, runtime = _build_engine( + raw, + Checkpoint.open(model_dir), + str(request.precision).lower(), + bool(request.verbose), + ) writer.set_header(family="timm_res2net", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_res2net/tests/test_sdk_metadata.py b/families/timm_res2net/tests/test_sdk_metadata.py index 87748571fe..116259db4a 100644 --- a/families/timm_res2net/tests/test_sdk_metadata.py +++ b/families/timm_res2net/tests/test_sdk_metadata.py @@ -26,7 +26,13 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): "mean": [0.5] * 3, "std": [0.25] * 3} monkeypatch.setattr(model, "_read_config", lambda _: raw) monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) - monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + engine_calls = [] + + def build_engine(*args): + engine_calls.append(args) + return b"plan", runtime + + monkeypatch.setattr(model, "_build_engine", build_engine) sections = {} headers = [] writer = SimpleNamespace(set_header=lambda **value: headers.append(value), @@ -38,8 +44,10 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): with pytest.raises(ValueError, match="vocabulary_id|label_names"): model.build(request, writer) assert not sections and not headers + assert not engine_calls return model.build(request, writer) + assert len(engine_calls) == 1 assert headers == [{"family": "timm_res2net", "task": "image_to_class_scores", "backend": "trt"}] assert sections["engine.plan"] == b"plan" assert sections["runtime.json"]["num_classes"] == 5 diff --git a/families/timm_senet/model.py b/families/timm_senet/model.py index ed0274e12c..00907a03fc 100644 --- a/families/timm_senet/model.py +++ b/families/timm_senet/model.py @@ -364,21 +364,21 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: _positive_int(request.max_sequence_length or 1, "max_sequence_length") model_dir = Path(request.model_dir) raw = _read_config(model_dir) - plan, runtime = _build_engine( - raw, - Checkpoint.open(model_dir), - str(request.precision).lower(), - bool(request.verbose), - ) vocabulary_id = raw.get("vocabulary_id", "") labels = raw.get("label_names", []) if not isinstance(vocabulary_id, str): raise ValueError("timm SENet vocabulary_id must be a string") if not isinstance(labels, list) or (labels and ( - len(labels) != runtime["num_classes"] + len(labels) != _preprocess_config(raw)["num_classes"] or any(not isinstance(label, str) or not label for label in labels) )): raise ValueError("timm SENet label_names must name every class") + plan, runtime = _build_engine( + raw, + Checkpoint.open(model_dir), + str(request.precision).lower(), + bool(request.verbose), + ) writer.set_header(family="timm_senet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_senet/tests/test_sdk_metadata.py b/families/timm_senet/tests/test_sdk_metadata.py index 4eb934ce51..3571d8f17b 100644 --- a/families/timm_senet/tests/test_sdk_metadata.py +++ b/families/timm_senet/tests/test_sdk_metadata.py @@ -26,7 +26,13 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): "mean": [0.5] * 3, "std": [0.25] * 3} monkeypatch.setattr(model, "_read_config", lambda _: raw) monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) - monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + engine_calls = [] + + def build_engine(*args): + engine_calls.append(args) + return b"plan", runtime + + monkeypatch.setattr(model, "_build_engine", build_engine) sections = {} headers = [] writer = SimpleNamespace(set_header=lambda **value: headers.append(value), @@ -38,8 +44,10 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): with pytest.raises(ValueError, match="vocabulary_id|label_names"): model.build(request, writer) assert not sections and not headers + assert not engine_calls return model.build(request, writer) + assert len(engine_calls) == 1 assert headers == [{"family": "timm_senet", "task": "image_to_class_scores", "backend": "trt"}] assert sections["engine.plan"] == b"plan" assert sections["runtime.json"]["num_classes"] == 5 diff --git a/families/timm_seresnet/model.py b/families/timm_seresnet/model.py index 7d252b5dcc..8aa894aba4 100644 --- a/families/timm_seresnet/model.py +++ b/families/timm_seresnet/model.py @@ -345,21 +345,21 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: _positive_int(request.max_sequence_length or 1, "max_sequence_length") model_dir = Path(request.model_dir) raw = _read_config(model_dir) - plan, runtime = _build_engine( - raw, - Checkpoint.open(model_dir), - str(request.precision).lower(), - bool(request.verbose), - ) vocabulary_id = raw.get("vocabulary_id", "") labels = raw.get("label_names", []) if not isinstance(vocabulary_id, str): raise ValueError("timm SE-ResNet vocabulary_id must be a string") if not isinstance(labels, list) or (labels and ( - len(labels) != runtime["num_classes"] + len(labels) != _preprocess_config(raw)["num_classes"] or any(not isinstance(label, str) or not label for label in labels) )): raise ValueError("timm SE-ResNet label_names must name every class") + plan, runtime = _build_engine( + raw, + Checkpoint.open(model_dir), + str(request.precision).lower(), + bool(request.verbose), + ) writer.set_header(family="timm_seresnet", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_seresnet/tests/test_sdk_metadata.py b/families/timm_seresnet/tests/test_sdk_metadata.py index 37a1f63f1a..e9ba621faa 100644 --- a/families/timm_seresnet/tests/test_sdk_metadata.py +++ b/families/timm_seresnet/tests/test_sdk_metadata.py @@ -26,7 +26,13 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): "mean": [0.5] * 3, "std": [0.25] * 3} monkeypatch.setattr(model, "_read_config", lambda _: raw) monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) - monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + engine_calls = [] + + def build_engine(*args): + engine_calls.append(args) + return b"plan", runtime + + monkeypatch.setattr(model, "_build_engine", build_engine) sections = {} headers = [] writer = SimpleNamespace(set_header=lambda **value: headers.append(value), @@ -38,8 +44,10 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): with pytest.raises(ValueError, match="vocabulary_id|label_names"): model.build(request, writer) assert not sections and not headers + assert not engine_calls return model.build(request, writer) + assert len(engine_calls) == 1 assert headers == [{"family": "timm_seresnet", "task": "image_to_class_scores", "backend": "trt"}] assert sections["engine.plan"] == b"plan" assert sections["runtime.json"]["num_classes"] == 5 diff --git a/families/timm_vgg/model.py b/families/timm_vgg/model.py index d31c9c9a14..0830b628e9 100644 --- a/families/timm_vgg/model.py +++ b/families/timm_vgg/model.py @@ -318,13 +318,13 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: precision = str(request.precision).lower() model = _TimmVggModel() weights = model.load_weights(str(model_dir), config, precision=precision) + runtime_source = model.get_bundle_config_overrides(config) plan = model.build_engine( config, weights, precision=precision, verbose=bool(request.verbose), ) - runtime_source = model.get_bundle_config_overrides(config) writer.set_header(family="timm_vgg", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_vgg/tests/test_model.py b/families/timm_vgg/tests/test_model.py index 72525dbba7..21afa403b2 100644 --- a/families/timm_vgg/tests/test_model.py +++ b/families/timm_vgg/tests/test_model.py @@ -243,14 +243,26 @@ def add_json(self, name, value): assert sections["runtime.json"]["vocabulary_id"] == metadata.get("vocabulary_id", "") -@pytest.mark.parametrize("labels", [["only-one"], ["a", "b", "", "d", "e"], 5]) -def test_build_rejects_incomplete_class_labels(tmp_path, monkeypatch, labels): +@pytest.mark.parametrize("metadata,message", [ + ({"label_names": ["only-one"]}, "label_names must name every class"), + ({"label_names": ["a", "b", "", "d", "e"]}, "label_names must name every class"), + ({"label_names": 5}, "label_names must name every class"), + ({"vocabulary_id": 5}, "vocabulary_id must be a string"), +]) +def test_build_rejects_invalid_class_metadata_before_engine(tmp_path, monkeypatch, metadata, message): _write_tiny_vgg(tmp_path) raw = json.loads((tmp_path / "config.json").read_text()) - raw["label_names"] = labels + raw.update(metadata) (tmp_path / "config.json").write_text(json.dumps(raw)) - monkeypatch.setattr(_TimmVggModel, "build_engine", lambda *args, **kwargs: b"plan") + engine_calls = [] + + def build_engine(*args, **kwargs): + engine_calls.append((args, kwargs)) + return b"plan" + + monkeypatch.setattr(_TimmVggModel, "build_engine", build_engine) request = BuildRequest(model_dir=tmp_path, output_path=tmp_path / "model.bundle", family="timm_vgg", task="image_to_class_scores", precision="fp32") - with pytest.raises(ValueError, match="label_names must name every class"): + with pytest.raises(ValueError, match=message): build_family(request, object()) + assert not engine_calls diff --git a/families/timm_xception/model.py b/families/timm_xception/model.py index 4c404569c9..63e8a1617e 100644 --- a/families/timm_xception/model.py +++ b/families/timm_xception/model.py @@ -362,21 +362,21 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: _positive_int(request.max_sequence_length or 1, "max_sequence_length") model_dir = Path(request.model_dir) raw = _read_config(model_dir) - plan, runtime = _build_engine( - raw, - Checkpoint.open(model_dir), - str(request.precision).lower(), - bool(request.verbose), - ) vocabulary_id = raw.get("vocabulary_id", "") labels = raw.get("label_names", []) if not isinstance(vocabulary_id, str): raise ValueError("timm Xception vocabulary_id must be a string") if not isinstance(labels, list) or (labels and ( - len(labels) != runtime["num_classes"] + len(labels) != _preprocess_config(raw)["num_classes"] or any(not isinstance(label, str) or not label for label in labels) )): raise ValueError("timm Xception label_names must name every class") + plan, runtime = _build_engine( + raw, + Checkpoint.open(model_dir), + str(request.precision).lower(), + bool(request.verbose), + ) writer.set_header(family="timm_xception", task=request.task, backend=request.backend) writer.add_bytes("engine.plan", plan) writer.add_json( diff --git a/families/timm_xception/tests/test_sdk_metadata.py b/families/timm_xception/tests/test_sdk_metadata.py index 832cc30fed..06ba6a2c02 100644 --- a/families/timm_xception/tests/test_sdk_metadata.py +++ b/families/timm_xception/tests/test_sdk_metadata.py @@ -26,7 +26,13 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): "mean": [0.5] * 3, "std": [0.25] * 3} monkeypatch.setattr(model, "_read_config", lambda _: raw) monkeypatch.setattr(model.Checkpoint, "open", lambda _: object()) - monkeypatch.setattr(model, "_build_engine", lambda *_: (b"plan", runtime)) + engine_calls = [] + + def build_engine(*args): + engine_calls.append(args) + return b"plan", runtime + + monkeypatch.setattr(model, "_build_engine", build_engine) sections = {} headers = [] writer = SimpleNamespace(set_header=lambda **value: headers.append(value), @@ -38,8 +44,10 @@ def test_builder_task_and_metadata(tmp_path, monkeypatch, metadata, invalid): with pytest.raises(ValueError, match="vocabulary_id|label_names"): model.build(request, writer) assert not sections and not headers + assert not engine_calls return model.build(request, writer) + assert len(engine_calls) == 1 assert headers == [{"family": "timm_xception", "task": "image_to_class_scores", "backend": "trt"}] assert sections["engine.plan"] == b"plan" assert sections["runtime.json"]["num_classes"] == 5