From ab7f8c08622afa22961b5a9ca398b4d9aea9c004 Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:24:42 -0700 Subject: [PATCH 1/7] ci(profiles): decouple family envs from base image Keep the reviewed runtime fingerprint limited to stable base-image inputs. Prepare exact family profile artifacts per model proof, install and verify them offline, and mount the resulting environments read-only. Allow family-owned source-build switches without adding model-specific behavior to shared CI. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- .dockerignore | 26 +- .github/scripts/build-python-profiles.py | 17 +- .../write-model-proof-fallback-report.py | 21 +- Dockerfile | 38 +- .../families/nemotron_h/MODEL.toml | 4 + .../tensorrt_model_connect/python_profiles.py | 99 +++++- tests/tools/test_e2e_python_profiles.py | 32 +- tests/tools/test_ensure_ci_docker_image.py | 115 +++--- tests/tools/test_github_actions_ci.py | 7 +- tests/tools/test_model_ci.py | 20 ++ tests/tools/test_model_proof_runner.py | 329 +++++++++++++++++- tools/ci/README.md | 65 +++- tools/ci/docker_image.py | 118 +------ tools/ci/model_proof.py | 298 +++++++++++++++- tools/ci/profile_downloader.py | 157 +++++++++ tools/model_ci.py | 1 + website/docs/extend/add-model-family.md | 35 ++ 17 files changed, 1110 insertions(+), 272 deletions(-) create mode 100644 tools/ci/profile_downloader.py diff --git a/.dockerignore b/.dockerignore index 48b4cd9f95..f3bfb79371 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,25 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# The repository Dockerfile copies only the declarative Python profile source -# and its image-build helper. Keep the daemon context stable and small even on -# long-lived self-hosted runners with large build/test artifacts. +# The repository Dockerfile installs the stable base toolchain and copies only +# its model-agnostic package downloader. Family profiles are prepared per proof. * -!python/ -!python/tensorrt_model_connect/ -!python/tensorrt_model_connect/__init__.py -!python/tensorrt_model_connect/python_profiles.py -!python/tensorrt_model_connect/python_profiles.toml -!python/tensorrt_model_connect/families/ -!python/tensorrt_model_connect/families/__init__.py -!python/tensorrt_model_connect/families/*/ -!python/tensorrt_model_connect/families/*/MODEL.toml -!python/tensorrt_model_connect/families/*/python_profile_requirements/ -!python/tensorrt_model_connect/families/*/python_profile_requirements/*.lock.txt -!python/tensorrt_model_connect/families/*/python_profile_verify.py -!.github/ -!.github/scripts/ -!.github/scripts/build-python-profiles.py - -**/__pycache__/ -**/*.py[cod] +!tools/ +!tools/ci/ +!tools/ci/profile_downloader.py diff --git a/.github/scripts/build-python-profiles.py b/.github/scripts/build-python-profiles.py index 0e1d2b607a..c2b15ab3db 100644 --- a/.github/scripts/build-python-profiles.py +++ b/.github/scripts/build-python-profiles.py @@ -1,12 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Materialize every declared Python profile during the CI image build.""" +"""Prepare exact-pinned Python profiles before network-disabled execution.""" from __future__ import annotations import importlib.util -import json import os import sys import types @@ -48,24 +47,12 @@ def main() -> None: raise SystemExit("no prebuilt Python profiles were declared") base_python = os.environ.get("TRTMC_BASE_PYTHON", "/opt/venv/bin/python") - resolved: dict[str, dict[str, str]] = {} for name in names: python = profile_api.resolve_profile_python(name, base_python) ready = Path(python).parent.parent / ".ready" if not ready.is_file(): raise SystemExit(f"profile {name!r} was not marked ready: {ready}") - resolved[name] = {"python": python, "ready": str(ready)} - - manifest = { - "schema_version": 1, - "profiles": resolved, - } - root = profile_api.profile_root() - (root / ".image-ready.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - print("prebuilt_python_profiles=" + ",".join(names)) + print("prepared_python_profiles=" + ",".join(names)) if __name__ == "__main__": diff --git a/.github/scripts/write-model-proof-fallback-report.py b/.github/scripts/write-model-proof-fallback-report.py index a5e8d4206c..49d7720d2a 100644 --- a/.github/scripts/write-model-proof-fallback-report.py +++ b/.github/scripts/write-model-proof-fallback-report.py @@ -9,6 +9,8 @@ import argparse import html import json +import os +import stat from pathlib import Path from typing import Sequence @@ -16,6 +18,8 @@ _DIAGNOSTIC_FILES = ( "host-error.log", "ci-image.log", + "python-profiles-prepare.log", + "python-profile-download.log", "console.log", "projection.stderr.log", "projection.json", @@ -23,6 +27,7 @@ "build.log", ) _MAX_DIAGNOSTIC_CHARS = 16_000 +_MAX_DIAGNOSTIC_BYTES = _MAX_DIAGNOSTIC_CHARS * 4 def _load_json(path: Path) -> dict[str, object]: @@ -37,14 +42,24 @@ def _diagnostics(root: Path) -> list[tuple[str, str]]: excerpts: list[tuple[str, str]] = [] for filename in _DIAGNOSTIC_FILES: path = root / filename - if not path.is_file(): + try: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except OSError: continue try: - text = path.read_text(encoding="utf-8", errors="replace") + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + continue + offset = max(0, metadata.st_size - _MAX_DIAGNOSTIC_BYTES) + os.lseek(descriptor, offset, os.SEEK_SET) + payload = os.read(descriptor, _MAX_DIAGNOSTIC_BYTES) except OSError: continue + finally: + os.close(descriptor) + text = payload.decode("utf-8", errors="replace")[-_MAX_DIAGNOSTIC_CHARS:] if text.strip(): - excerpts.append((filename, text[-_MAX_DIAGNOSTIC_CHARS:])) + excerpts.append((filename, text)) return excerpts diff --git a/Dockerfile b/Dockerfile index e5e32762ca..b477e08623 100644 --- a/Dockerfile +++ b/Dockerfile @@ -135,44 +135,20 @@ RUN pip install --force-reinstall \ # reference inference on the system cuBLAS instead of pip-installed CUDA libs. ENV LD_PRELOAD=/usr/local/cuda/lib64/libcublas.so.13 +# This model-agnostic downloader is part of the reviewed base runtime. It may +# fetch exact public PyPI artifacts, but never imports or builds package code. +COPY tools/ci/profile_downloader.py /opt/trtmc-profile-downloader.py + # Coverage tooling verification (run inside container): # python3 -m coverage --version && pytest --version && \ # python3 -m pytest --help | grep -- '--cov' && \ # gcovr --version && lcov --version && genhtml --version -# Build every declarative Python execution profile while network access is -# available. Family-owned declarations, lock files, and verification scripts -# are the package source of truth; python_profiles.py rejects non-exact pins and -# verifies every installed distribution before marking a profile ready. -FROM ci-common-base AS python-profile-builder - -ENV TRTMC_PYTHON_PROFILE_ROOT=/opt/trtmc-python-profiles -# sphn publishes no aarch64 wheel. Keep its Rust build toolchain in this -# throwaway builder stage; the final development stage receives only the -# verified profile. -RUN apt-get update && \ - apt-get install -y --no-install-recommends cargo rustc && \ - rm -rf /var/lib/apt/lists/* && \ - pip install "maturin==1.14.1" -# Avoid compiling profile-local CUDA extensions for every architecture known -# to a GPU-less Docker build. Keep 10.0 as the GB300 target. -COPY python/tensorrt_model_connect /opt/trtmc-profile-source/tensorrt_model_connect -COPY .github/scripts/build-python-profiles.py /opt/trtmc-build-python-profiles.py -RUN python3 /opt/trtmc-build-python-profiles.py \ - && chmod -R a+rX /opt/trtmc-python-profiles - -# Do not retain the full builder source tree in the development image. Only the -# verified virtual environments cross the stage boundary, so sibling model -# implementations cannot satisfy imports in an isolated source projection. +# Keep the reusable runtime independent of family-owned Python environments. +# Online CI preparation materializes the selected family's exact-pinned +# profiles before a network-disabled proof and mounts them read-only there. FROM ci-common-base AS ci-common -COPY --from=python-profile-builder \ - /opt/trtmc-python-profiles /opt/trtmc-python-profiles -ENV TRTMC_PYTHON_PROFILE_ROOT=/opt/trtmc-python-profiles -# Execution-profile environments are part of the dev-image contract. Rebuild -# the image after changing their lock or verification files. -ENV TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1 - # Keep the reusable common layer independent of every TensorRT release. The # version overlay below is the only stage allowed to add bindings, headers, or # native runtime libraries. diff --git a/python/tensorrt_model_connect/families/nemotron_h/MODEL.toml b/python/tensorrt_model_connect/families/nemotron_h/MODEL.toml index 544594edde..b32d4be6a4 100644 --- a/python/tensorrt_model_connect/families/nemotron_h/MODEL.toml +++ b/python/tensorrt_model_connect/families/nemotron_h/MODEL.toml @@ -7,6 +7,10 @@ module = "plugin" python_profile_specs = [ "nemotron_h_reference|families/nemotron_h/python_profile_requirements/nemotron_h_reference.lock.txt|families/nemotron_h/python_profile_verify.py|true", ] +python_profile_build_environment = [ + "nemotron_h_reference|CAUSAL_CONV1D_FORCE_BUILD|TRUE", + "nemotron_h_reference|MAMBA_FORCE_BUILD|TRUE", +] default_execution_profiles = [ "reference|nemotron_h_reference", ] diff --git a/python/tensorrt_model_connect/python_profiles.py b/python/tensorrt_model_connect/python_profiles.py index d2747c3df6..6151bcaa1e 100644 --- a/python/tensorrt_model_connect/python_profiles.py +++ b/python/tensorrt_model_connect/python_profiles.py @@ -45,6 +45,25 @@ re.IGNORECASE, ) _PROFILE_NAME_RE = re.compile(r"[a-z][a-z0-9_]*") +_BUILD_ENVIRONMENT_NAME_RE = re.compile(r"[A-Z][A-Z0-9_]*") +_FORBIDDEN_BUILD_ENVIRONMENT_NAMES = { + "HOME", + "LD_LIBRARY_PATH", + "LD_PRELOAD", + "PATH", + "PYTHONHOME", + "PYTHONPATH", +} +_FORBIDDEN_BUILD_ENVIRONMENT_PREFIXES = ( + "AWS_", + "AZURE_", + "GIT_", + "GOOGLE_", + "NVIDIA_", + "PIP_", + "SSH_", + "TRTMC_", +) _REGISTRY_KEYS = { "version", "profiles", @@ -55,6 +74,7 @@ _VENV_PROFILE_KEYS = { "kind", "prebuild", + "build_environment", "requirements", "system_site_packages", "verification_script", @@ -153,6 +173,7 @@ def family_python_profile_specs() -> dict[str, dict[str, object]]: with manifest.open("rb") as stream: raw = tomllib.load(stream) family_id = raw.get("id") or raw.get("plugin") or manifest.parent.name + family_profile_names: set[str] = set() raw_specs = raw.get("python_profile_specs", []) if not isinstance(raw_specs, list): raise ValueError( @@ -192,6 +213,49 @@ def family_python_profile_specs() -> dict[str, dict[str, object]]: "system_site_packages": system_site_packages, "prebuild": prebuild, } + family_profile_names.add(name) + raw_build_environment = raw.get("python_profile_build_environment", []) + if not isinstance(raw_build_environment, list): + raise ValueError( + f"python_profile_build_environment for family {family_id} must be a list" + ) + for entry in raw_build_environment: + if not isinstance(entry, str): + raise ValueError( + f"python_profile_build_environment for family {family_id} " + "must contain strings" + ) + parts = [part.strip() for part in entry.split("|", 2)] + if len(parts) != 3 or any(not part for part in parts): + raise ValueError( + f"Invalid python_profile_build_environment entry {entry!r} " + f"for family {family_id}; expected 'profile|NAME|value'" + ) + profile, name, value = parts + if profile not in family_profile_names: + raise ValueError( + f"python_profile_build_environment selects undeclared profile " + f"{profile!r} for family {family_id}" + ) + if ( + _BUILD_ENVIRONMENT_NAME_RE.fullmatch(name) is None + or name in _FORBIDDEN_BUILD_ENVIRONMENT_NAMES + or name.startswith(_FORBIDDEN_BUILD_ENVIRONMENT_PREFIXES) + ): + raise ValueError( + f"Python profile {profile!r} has unsafe build environment name {name!r}" + ) + if len(value) > 1024 or "\x00" in value or "\n" in value or "\r" in value: + raise ValueError( + f"Python profile {profile!r} has an unsafe build environment value" + ) + build_environment = dict(profiles[profile].get("build_environment", {})) + if name in build_environment: + raise ValueError( + f"Python profile {profile!r} declares build environment {name!r} twice" + ) + build_environment[name] = value + profiles[profile]["build_environment"] = build_environment return profiles @@ -260,6 +324,23 @@ def _validate_python_profile_registry(registry: Mapping[str, Any]) -> None: raise ValueError( f"Execution profile {name!r} field {field} must be a bool" ) + build_environment = raw_spec.get("build_environment", {}) + if not isinstance(build_environment, Mapping) or any( + not isinstance(name, str) + or _BUILD_ENVIRONMENT_NAME_RE.fullmatch(name) is None + or name in _FORBIDDEN_BUILD_ENVIRONMENT_NAMES + or name.startswith(_FORBIDDEN_BUILD_ENVIRONMENT_PREFIXES) + or not isinstance(value, str) + or not value + or len(value) > 1024 + or "\x00" in value + or "\n" in value + or "\r" in value + for name, value in build_environment.items() + ): + raise ValueError( + f"Execution profile {name!r} build_environment must contain safe strings" + ) requirements = raw_spec.get("requirements") if type(requirements) is not str: raise ValueError( @@ -334,7 +415,7 @@ def _validate_python_profile_registry(registry: Mapping[str, Any]) -> None: def prebuilt_python_profile_names( registry: Mapping[str, Any] | None = None, ) -> tuple[str, ...]: - """Return non-default profiles that belong in the shared CI image.""" + """Return non-default profiles prepared before network-disabled execution.""" selected = ( registry if registry is not None else load_python_profile_registry() ) @@ -669,6 +750,10 @@ def _materialize_venv_profile( ) verification_script = _read_package_text(verification_script_file).strip() system_site_packages = bool(spec.get("system_site_packages", True)) + build_environment = { + str(name): str(value) + for name, value in dict(spec.get("build_environment", {})).items() + } hash_input = "\n".join( [ @@ -678,6 +763,7 @@ def _materialize_venv_profile( requirements_text, verification_script, f"system_site_packages={int(system_site_packages)}", + json.dumps(build_environment, separators=(",", ":"), sort_keys=True), ] ).encode("utf-8") profile_hash = hashlib.sha256(hash_input).hexdigest()[:12] @@ -688,15 +774,14 @@ def _materialize_venv_profile( ready_path = env_dir / ".ready" lock_path = root / f"{profile_name}-{profile_hash}.lock" - # Model-proof containers mount the source read-only and disable networking. - # A matching image-baked profile therefore needs no writable lock or cache. + # Network-disabled proofs mount a separately prepared profile root read-only. if ready_path.is_file() and python_path.is_file(): return str(python_path.absolute()) if _prebuilt_only(): raise RuntimeError( f"Execution profile {profile_name!r} is not prebuilt for this source " - f"at {env_dir}. The CI image is stale or incomplete; rebuild it from " - "the current Dockerfile and declarative profile locks." + f"at {env_dir}. Prepare the declared profiles before entering the " + "network-disabled execution lane." ) root.mkdir(parents=True, exist_ok=True) @@ -726,6 +811,8 @@ def _materialize_venv_profile( _write_base_site_packages_overlay(base_python, str(tmp_python)) if requirements_text.strip(): + install_environment = _profile_install_environment() + install_environment.update(build_environment) _run_profile_command( [ str(tmp_python), @@ -741,7 +828,7 @@ def _materialize_venv_profile( ], description=f"install Python profile {profile_name!r}", timeout=_PROFILE_INSTALL_TIMEOUT_SECONDS, - env=_profile_install_environment(), + env=install_environment, ) _verify_exact_requirements( diff --git a/tests/tools/test_e2e_python_profiles.py b/tests/tools/test_e2e_python_profiles.py index 1105998d14..6726806c48 100644 --- a/tests/tools/test_e2e_python_profiles.py +++ b/tests/tools/test_e2e_python_profiles.py @@ -151,6 +151,9 @@ def test_resolve_profile_python_materializes_declared_venv(monkeypatch, tmp_path == python ) assert created == ["custom"] + monkeypatch.setenv(shared_profiles.PREBUILT_ONLY_ENV, "1") + assert shared_profiles.resolve_profile_python("custom", sys.executable) == python + assert created == ["custom"] def test_profile_source_builds_use_a_safe_default_job_limit(monkeypatch, tmp_path): @@ -183,12 +186,14 @@ def run_command(cmd, *, description, timeout=1800, **kwargs): "requirements": str(requirements), "system_site_packages": False, "verification_script": "print('verified')", + "build_environment": {"DEMO_FORCE_BUILD": "TRUE"}, }, sys.executable, ) install = next(call for call in commands if call[1].startswith("install ")) assert install[3]["env"]["MAX_JOBS"] == "4" + assert install[3]["env"]["DEMO_FORCE_BUILD"] == "TRUE" assert "PYTHONPATH" not in install[3]["env"] assert install[2] == 7200 verify = next(call for call in commands if call[1].startswith("verify ")) @@ -269,6 +274,28 @@ def test_family_profile_registry_is_fully_exact_pinned(): ) pins = shared_profiles._exact_pinned_requirements(requirements) assert pins, name + assert profiles["nemotron_h_reference"]["build_environment"] == { + "CAUSAL_CONV1D_FORCE_BUILD": "TRUE", + "MAMBA_FORCE_BUILD": "TRUE", + } + + +def test_profile_build_environment_rejects_package_source_overrides() -> None: + with pytest.raises(ValueError, match="safe strings"): + shared_profiles._validate_python_profile_registry( + { + "version": 1, + "profiles": { + "base": {"kind": "passthrough"}, + "unsafe": { + "kind": "venv", + "requirements": "python_profile_requirements/reference_common.lock.txt", + "verification_script": "pass", + "build_environment": {"PIP_INDEX_URL": "https://example.invalid"}, + }, + } + } + ) def test_profile_contract_has_one_family_owned_source_of_truth() -> None: @@ -287,13 +314,14 @@ def test_profile_contract_has_one_family_owned_source_of_truth() -> None: assert merged_names == shared_names | family_names -def test_lazy_profiles_are_excluded_from_the_shared_ci_image() -> None: +def test_lazy_profiles_are_excluded_from_offline_preparation() -> None: registry = shared_profiles.load_python_profile_registry() prebuilt = shared_profiles.prebuilt_python_profile_names(registry) assert "personaplex_full_duplex_evaluator" not in prebuilt assert "reference_common" in prebuilt + def test_profile_lock_rejects_non_exact_or_duplicate_requirements(): with pytest.raises(ValueError, match="exact name==version pins"): shared_profiles._exact_pinned_requirements("transformers>=4.48\n") @@ -450,7 +478,7 @@ def test_prebuilt_only_profile_fails_before_creating_a_runtime_cache( }, ) - with pytest.raises(RuntimeError, match="CI image is stale or incomplete"): + with pytest.raises(RuntimeError, match="Prepare the declared profiles"): shared_profiles.resolve_profile_python("custom", sys.executable) assert not profile_root.exists() diff --git a/tests/tools/test_ensure_ci_docker_image.py b/tests/tools/test_ensure_ci_docker_image.py index 89b1ff46c9..4f559fb646 100644 --- a/tests/tools/test_ensure_ci_docker_image.py +++ b/tests/tools/test_ensure_ci_docker_image.py @@ -266,6 +266,7 @@ def _write_profile_fingerprint_repo(tmp_path: Path) -> tuple[Path, Path, Path]: shutil.copytree(REPO_ROOT / ".github" / "scripts", repo_root / ".github" / "scripts") shutil.copytree(REPO_ROOT / "tools" / "ci", repo_root / "tools" / "ci") shutil.copy2(REPO_ROOT / "tools" / "__init__.py", repo_root / "tools" / "__init__.py") + shutil.copy2(REPO_ROOT / ".dockerignore", repo_root / ".dockerignore") package_root = repo_root / "python" / "tensorrt_model_connect" families_root = package_root / "families" @@ -464,7 +465,7 @@ def test_matching_image_is_fully_validated_once_per_workflow_run(tmp_path: Path) assert "reused from this workflow run's verified image" in result.stdout -def test_missing_prebuilt_profiles_rebuilds_the_image(tmp_path: Path) -> None: +def test_base_image_ignores_family_profile_inventory(tmp_path: Path) -> None: bootstrap_result, bootstrap_env, bootstrap_log = _run_ensure_script( tmp_path / "bootstrap", existing_images={}, @@ -483,8 +484,8 @@ def test_missing_prebuilt_profiles_rebuilds_the_image(tmp_path: Path) -> None: ) assert result.returncode == 0, result.stderr - assert f"-t {resolved_image}" in docker_log - assert "prebuilt Python profiles differ" in result.stdout + assert f"-t {resolved_image}" not in docker_log + assert f"CI Docker image '{resolved_image}' already matches" in result.stdout def test_tensorrt_overlay_contract_reports_each_mismatch(tmp_path: Path) -> None: @@ -537,13 +538,13 @@ def test_source_contract_describes_parameterized_tensorrt_overlay(tmp_path: Path tensorrt_apt_version="11.2.1.2-1+cuda13.3", ) assert selected_contract["schema_version"] == 1 - assert selected_contract["environment_contract_version"] == 2 + assert selected_contract["environment_contract_version"] == 3 assert ( selected_contract["common_input_fingerprint"] == default_contract["common_input_fingerprint"] ) assert selected_contract["input_fingerprint"] != default_contract["input_fingerprint"] - assert selected_contract["python_profiles"] == ["demo", "reference_common"] + assert selected_contract["python_profiles"] == [] assert selected_contract["tensorrt"] == { "version": "11.2.1.2", "apt_version": "11.2.1.2-1+cuda13.3", @@ -563,7 +564,7 @@ def test_source_contract_describes_parameterized_tensorrt_overlay(tmp_path: Path } -def test_source_contract_loads_profiles_without_ambient_pythonpath(tmp_path: Path) -> None: +def test_source_contract_ignores_profiles_without_ambient_pythonpath(tmp_path: Path) -> None: repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) environment = os.environ.copy() environment.pop("PYTHONPATH", None) @@ -585,7 +586,7 @@ def test_source_contract_loads_profiles_without_ambient_pythonpath(tmp_path: Pat ) assert result.returncode == 0, result.stderr - assert result.stdout.strip() == "demo,reference_common" + assert result.stdout.strip() == "" def test_image_contract_cli_emits_canonical_contract_json(tmp_path: Path) -> None: @@ -611,7 +612,8 @@ def test_image_contract_cli_emits_canonical_contract_json(tmp_path: Path) -> Non assert result.returncode == 0, result.stderr contract = json.loads(result.stdout) - assert contract["environment_contract_version"] == 2 + assert contract["environment_contract_version"] == 3 + assert contract["python_profiles"] == [] assert contract["tensorrt"]["version"] == "11.2.1.2" assert contract["tensorrt"]["apt_version"] == "11.2.1.2-1+cuda13.3" @@ -636,7 +638,6 @@ def test_validate_image_contract_returns_the_verified_source_contract( "MODELOPT_VERSION": expected.modelopt, "NLOHMANN_JSON_HEADER": "present", "NEMO_PROMPT_RNNT": "available", - "PYTHON_PROFILES": expected.python_profiles, } monkeypatch.setattr(manager, "_query_fingerprint", lambda _: expected.fingerprint) monkeypatch.setattr(manager, "_query_versions", lambda _: actual) @@ -696,48 +697,20 @@ def test_source_contract_rejects_mixed_tensorrt_overlay_versions(tmp_path: Path) ) -def test_source_contract_rechecks_profile_asset_containment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) - outside = tmp_path / "outside.lock.txt" - outside.write_text("demo==1.0\n", encoding="utf-8") - manager = DockerImageManager(repo_root) - monkeypatch.setattr( - manager, - "_load_profile_registry", - lambda: ( - { - "version": 1, - "profiles": { - "demo": { - "kind": "venv", - "prebuild": True, - "requirements": str(outside), - } - }, - }, - ("demo",), - ), - ) - - with pytest.raises(CiError, match="unsafe requirements path"): - manager.source_contract() - - -def test_profile_sources_are_fingerprinted_and_repo_is_the_build_context() -> None: +def test_only_base_sources_are_fingerprinted_and_repo_is_the_build_context() -> None: script = SCRIPT.read_text(encoding="utf-8") assert "class DockerImageManager" in script assert "semantic_fingerprint" in script - assert 'b"python-profile-registry\\0"' in script - assert "assets: set[Path]" in script + assert 'b"ci-base-runtime\\0"' in script + assert "assets: set[Path]" not in script assert 'Path("tools/ci/process.py")' not in script - assert 'package_root / "python_profiles.py"' in script + assert 'Path("tools/ci/profile_downloader.py")' in script + assert 'package_root / "python_profiles.py"' not in script assert '"-f"' in script assert "str(self.config.dockerfile)" in script assert '"."' in script - assert "profile builder source leaked into the runtime image" in script + assert "profile builder source leaked into the runtime image" not in script assert '"--user"' in script assert '"65534:65534"' in script assert '"--read-only"' in script @@ -747,7 +720,7 @@ def test_profile_sources_are_fingerprinted_and_repo_is_the_build_context() -> No assert "source_contract_json" in script -def test_profile_fingerprint_ignores_manifest_comments_and_ownership_fields( +def test_base_fingerprint_ignores_manifest_comments_and_ownership_fields( tmp_path: Path, ) -> None: repo_root, manifest, _ = _write_profile_fingerprint_repo(tmp_path) @@ -774,7 +747,7 @@ def test_profile_fingerprint_ignores_manifest_comments_and_ownership_fields( assert ownership_changed == baseline -def test_profile_fingerprint_ignores_unrelated_family_loader_changes( +def test_base_fingerprint_ignores_unrelated_family_loader_changes( tmp_path: Path, ) -> None: repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) @@ -860,7 +833,7 @@ def test_source_contract_does_not_execute_or_fingerprint_family_loader( assert changed["input_fingerprint"] == baseline["input_fingerprint"] -def test_profile_fingerprint_ignores_registry_comments(tmp_path: Path) -> None: +def test_base_fingerprint_ignores_profile_registry_comments(tmp_path: Path) -> None: repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) baseline = _resolved_image_for_repo(tmp_path / "baseline", repo_root) @@ -874,7 +847,7 @@ def test_profile_fingerprint_ignores_registry_comments(tmp_path: Path) -> None: assert changed == baseline -def test_profile_fingerprint_ignores_lazy_profile_declarations(tmp_path: Path) -> None: +def test_base_fingerprint_ignores_lazy_profile_declarations(tmp_path: Path) -> None: repo_root, manifest, _ = _write_profile_fingerprint_repo(tmp_path) baseline = _resolved_image_for_repo(tmp_path / "baseline", repo_root) @@ -890,7 +863,7 @@ def test_profile_fingerprint_ignores_lazy_profile_declarations(tmp_path: Path) - assert changed == baseline -def test_profile_fingerprint_changes_for_semantic_profile_declaration( +def test_base_fingerprint_ignores_semantic_profile_declaration( tmp_path: Path, ) -> None: repo_root, manifest, _ = _write_profile_fingerprint_repo(tmp_path) @@ -906,10 +879,21 @@ def test_profile_fingerprint_changes_for_semantic_profile_declaration( ) changed = _resolved_image_for_repo(tmp_path / "profile-change", repo_root) - assert changed != baseline + assert changed == baseline + + manifest.write_text( + manifest.read_text(encoding="utf-8") + + 'python_profile_build_environment = ["demo|DEMO_FORCE_BUILD|TRUE"]\n', + encoding="utf-8", + ) + environment_changed = _resolved_image_for_repo( + tmp_path / "profile-environment-change", + repo_root, + ) + assert environment_changed == baseline -def test_profile_fingerprint_changes_for_referenced_profile_asset_content( +def test_base_fingerprint_ignores_referenced_profile_asset_content( tmp_path: Path, ) -> None: repo_root, _, requirements = _write_profile_fingerprint_repo(tmp_path) @@ -918,19 +902,19 @@ def test_profile_fingerprint_changes_for_referenced_profile_asset_content( requirements.write_text("demo-package==1.0.1\n", encoding="utf-8") changed = _resolved_image_for_repo(tmp_path / "asset-change", repo_root) - assert changed != baseline + assert changed == baseline @pytest.mark.parametrize( "relative_path", ( - Path("Dockerfile"), Path(".github/scripts/build-python-profiles.py"), Path("python/tensorrt_model_connect/python_profiles.py"), + Path("python/tensorrt_model_connect/python_profiles.toml"), Path("python/tensorrt_model_connect/families/demo/verify.py"), ), ) -def test_profile_fingerprint_changes_for_every_baked_recipe_input( +def test_base_fingerprint_ignores_profile_preparation_inputs( tmp_path: Path, relative_path: Path, ) -> None: @@ -944,6 +928,31 @@ def test_profile_fingerprint_changes_for_every_baked_recipe_input( changed = _resolved_image_for_repo(tmp_path / "recipe-change", repo_root) + assert changed == baseline + + +@pytest.mark.parametrize( + "relative_path", + ( + Path("Dockerfile"), + Path(".dockerignore"), + Path("tools/ci/profile_downloader.py"), + ), +) +def test_base_fingerprint_changes_for_base_recipe_inputs( + tmp_path: Path, + relative_path: Path, +) -> None: + repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) + baseline = _resolved_image_for_repo(tmp_path / "baseline", repo_root) + target = repo_root / relative_path + target.write_text( + target.read_text(encoding="utf-8") + "\n# Base environment change.\n", + encoding="utf-8", + ) + + changed = _resolved_image_for_repo(tmp_path / "base-change", repo_root) + assert changed != baseline diff --git a/tests/tools/test_github_actions_ci.py b/tests/tools/test_github_actions_ci.py index 2773417c41..c672d3d10e 100644 --- a/tests/tools/test_github_actions_ci.py +++ b/tests/tools/test_github_actions_ci.py @@ -709,11 +709,11 @@ def test_source_ci_image_uses_common_and_parameterized_tensorrt_overlay() -> Non assert "ghcr.io" not in dockerfile assert "TENSORRT_SDK_IMAGE" not in dockerfile assert "/opt/tensorrt/python" not in dockerfile + assert "COPY tools/ci/profile_downloader.py /opt/trtmc-profile-downloader.py" in dockerfile from_lines = [line for line in dockerfile.splitlines() if line.startswith("FROM ")] assert from_lines == [ "FROM ${CUDA_IMAGE} AS ci-common-base", - "FROM ci-common-base AS python-profile-builder", "FROM ci-common-base AS ci-common", "FROM ci-common AS ci-runtime", ] @@ -721,8 +721,9 @@ def test_source_ci_image_uses_common_and_parameterized_tensorrt_overlay() -> Non common = dockerfile.split("FROM ci-common-base AS ci-common", maxsplit=1)[1].split( "FROM ci-common AS ci-runtime", maxsplit=1 )[0] - assert "COPY --from=python-profile-builder" in common - assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1" in common + assert "COPY --from=python-profile-builder" not in common + assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY" not in dockerfile + assert "/opt/trtmc-python-profiles" not in dockerfile assert 'find_spec("tensorrt") is None' in common assert "NvInferVersion.h" in common assert "NvOnnxParser.h" in common diff --git a/tests/tools/test_model_ci.py b/tests/tools/test_model_ci.py index 49c88e350f..326879b2c5 100644 --- a/tests/tools/test_model_ci.py +++ b/tests/tools/test_model_ci.py @@ -138,6 +138,11 @@ def _make_repo( _write(repo, "tests/runtime_strategy_matrix.yaml", "strategies: []\n") for source in sorted((REPO_ROOT / "tools/ci").glob("*.py")): _write(repo, f"tools/ci/{source.name}", f"# projected CI module: {source.name}\n") + _write( + repo, + ".github/scripts/build-python-profiles.py", + "#!/usr/bin/env python3\n", + ) _write( repo, ".github/scripts/write-model-proof-fallback-report.py", @@ -1228,6 +1233,7 @@ def test_projection_contains_only_selected_model_and_stable_git_blobs( assert fallback.is_file() assert not os.access(fallback, os.X_OK) for report_path in ( + ".github/scripts/build-python-profiles.py", "scripts/generate_e2e_report.py", "scripts/generate_e2e_report_assets/e2e_report.css", "scripts/generate_e2e_report_assets/e2e_report.js", @@ -1304,6 +1310,8 @@ def test_projection_includes_only_the_selected_family_adapter_subtrees( ) -> None: repo, _ = _make_repo(tmp_path) selected_paths = ( + "python/tensorrt_model_connect/families/model_b/python_profile_requirements/reference.lock.txt", + "python/tensorrt_model_connect/families/model_b/python_profile_verify.py", "python/tensorrt_model_connect/families/model_b/optimized_adapter/adapter.py", "python/tensorrt_model_connect/families/model_b/optimized_adapter/dependency.lock", "python/tensorrt_model_connect/families/model_b/optimized_adapter/profiles/example.toml", @@ -1315,6 +1323,18 @@ def test_projection_includes_only_the_selected_family_adapter_subtrees( _write(repo, path, "# selected model adapter\n") for path in sibling_paths: _write(repo, path, "# sibling model adapter\n") + for model in ("model_a", "model_b"): + manifest = repo / f"python/tensorrt_model_connect/families/{model}/MODEL.toml" + manifest.write_text( + manifest.read_text(encoding="utf-8") + + "python_profile_specs = [\n" + + ( + f' "{model}_reference|families/{model}/python_profile_requirements/' + f'reference.lock.txt|families/{model}/python_profile_verify.py|true",\n' + ) + + "]\n", + encoding="utf-8", + ) generic_host = "src/runtime/providers/optimized_runtime_host.cpp" _write(repo, generic_host, "// shared provider host\n") revision = _commit(repo, "add model-owned adapters") diff --git a/tests/tools/test_model_proof_runner.py b/tests/tools/test_model_proof_runner.py index 60d13d459b..7d7d65f5fd 100644 --- a/tests/tools/test_model_proof_runner.py +++ b/tests/tools/test_model_proof_runner.py @@ -7,6 +7,8 @@ from collections.abc import Callable, Iterator import fcntl +import hashlib +import io import json import os import re @@ -23,10 +25,17 @@ import pytest from tools.ci import gpu_lease as gpu_lease_module +from tools.ci import profile_downloader from tools.ci.context import CiContext from tools.ci.gpu_lease import GpuLease from tools.ci.model_reference_cache import ModelReferenceCacheWarmer -from tools.ci.model_proof import ModelProofRequest, ModelProofRunner, ModelReferenceCache +from tools.ci.model_proof import ( + PREPARED_PROFILE_ROOT, + PROFILE_PACKAGES_ROOT, + ModelProofRequest, + ModelProofRunner, + ModelReferenceCache, +) from tools.ci.model_proof_inner import ( ModelProofInnerPipeline, _classify_e2e_proof_kinds, @@ -111,6 +120,27 @@ def _write_successful_fake_docker(tmp_path: Path) -> tuple[Path, Path]: " exit 0\n" " ;;\n" " run)\n" + ' if [[ " $* " == *" /opt/trtmc-profile-downloader.py "* ]]; then\n' + " exit 0\n" + " fi\n" + ' if [[ " $* " == *" /src/.github/scripts/build-python-profiles.py "* ]]; then\n' + ' profile_root=""\n' + ' for argument in "$@"; do\n' + ' case "$argument" in\n' + ' type=bind,src=*,dst=/opt/trtmc-python-profiles)\n' + ' profile_root="${argument#type=bind,src=}"\n' + ' profile_root="${profile_root%,dst=/opt/trtmc-python-profiles}"\n' + " ;;\n" + " esac\n" + " done\n" + ' [ -n "$profile_root" ] || exit 95\n' + ' profile="$profile_root/reference_common-fake"\n' + ' mkdir -p "$profile/bin"\n' + ' ln -s /opt/venv/bin/python "$profile/bin/python"\n' + ' printf \'%s\\n\' \'profile=reference_common\' > "$profile/.ready"\n' + ' printf \'%s\\n\' \'{"schema_version":1,"profiles":{"reference_common":{"python":"/opt/trtmc-python-profiles/reference_common-fake/bin/python","ready":"/opt/trtmc-python-profiles/reference_common-fake/.ready"}}}\' > "$profile_root/.prepared-profiles.json"\n' + " exit 0\n" + " fi\n" ' if [[ " $* " == *" /src/scripts/warm_hf_cache.py "* ]]; then\n' ' mkdir -p "$FAKE_ARTIFACTS"\n' ' if [ "${FAKE_CACHE_EVIDENCE_MODE:-valid}" = escape ]; then\n' @@ -274,12 +304,17 @@ def _fake_proof_environment( return env -def _run_fake_proof(env: dict[str, str], output: Path) -> subprocess.CompletedProcess[str]: +def _run_fake_proof( + env: dict[str, str], + output: Path, + *, + model: str = "convbert", +) -> subprocess.CompletedProcess[str]: return subprocess.run( [ *RUNNER_COMMAND, "--model", - "convbert", + model, "--revision", "HEAD", "--output-dir", @@ -1774,6 +1809,281 @@ def test_model_proof_report_assets_are_inside_the_positive_projection() -> None: assert path.is_file(), path +def test_profile_preparation_uses_a_minimal_online_boundary( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + projection = tmp_path / "projection" + family = projection / "python/tensorrt_model_connect/families/demo" + family.mkdir(parents=True) + (family / "MODEL.toml").write_text( + 'id = "demo"\n' + 'python_profile_specs = [' + '"demo|families/demo/requirements.lock.txt|families/demo/verify.py|true"' + "]\n", + encoding="utf-8", + ) + (family / "requirements.lock.txt").write_text( + "demo-package==1.0.0\n", + encoding="utf-8", + ) + (family / "verify.py").write_text("import demo_package\n", encoding="utf-8") + package_root = projection / "python/tensorrt_model_connect" + (package_root / "python_profiles.toml").write_text( + 'version = 1\n[profiles.base]\nkind = "passthrough"\n', + encoding="utf-8", + ) + profiles = tmp_path / "python-profiles" + profiles.mkdir() + packages = tmp_path / "python-profile-packages" + packages.mkdir() + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + runner = ModelProofRunner( + CiContext(REPO_ROOT, {}), + ModelProofRequest(model="demo"), + ) + runner.artifacts_dir = artifacts + monkeypatch.setattr( + runner.context, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=0), + ) + commands: list[list[object]] = [] + + def prepare(command: list[object], _log: Path, **_kwargs) -> int: + commands.append(command) + return 0 + + monkeypatch.setattr(runner, "_run_logged", prepare) + + runner._prepare_python_profiles( + projection, + profiles, + packages, + "qualified-base@sha256:test", + ) + + assert len(commands) == 2 + download = " ".join(map(str, commands[0])) + install = " ".join(map(str, commands[1])) + assert "--network bridge" in download + assert "--runtime runc" in download + assert "/opt/trtmc-profile-downloader.py" in download + assert "demo-package==1.0.0" in download + assert f"src={packages},dst={PROFILE_PACKAGES_ROOT}" in download + assert "src={projection}" not in download + assert "NVIDIA_VISIBLE_DEVICES=void" in download + assert "CUDA_VISIBLE_DEVICES=" in download + + assert "--network none" in install + assert "--runtime runc" in install + assert "--read-only" in install + assert "--cap-drop ALL" in install + assert "--security-opt no-new-privileges" in install + assert "--ipc private" in install + assert "PIP_CONFIG_FILE=/dev/null" in install + assert "PIP_FIND_LINKS=/opt/trtmc-python-profile-packages" in install + assert "PIP_NO_INDEX=1" in install + assert f"src={projection},dst=/src,readonly" in install + assert f"src={profiles},dst={PREPARED_PROFILE_ROOT}" in install + assert f"src={packages},dst={PROFILE_PACKAGES_ROOT},readonly" in install + assert f"dst={PREPARED_PROFILE_ROOT},readonly" not in install + for command in (download, install): + assert "--gpus" not in command + assert "HF_TOKEN" not in command + assert "/var/run/docker.sock" not in command + assert "dst=/work" not in command + assert "dst=/artifacts" not in command + + +def test_profile_owning_family_runs_download_prepare_then_offline_proof( + tmp_path: Path, +) -> None: + fake_bin, docker_log = _write_successful_fake_docker(tmp_path) + output = tmp_path / "proof" + environment = _fake_proof_environment(tmp_path, fake_bin, docker_log, output) + + result = _run_fake_proof(environment, output, model="chronos_bolt") + + assert result.returncode == 0, result.stdout + result.stderr + runs = [ + line + for line in docker_log.read_text(encoding="utf-8").splitlines() + if line.startswith("run ") + ] + download_index = next( + index for index, command in enumerate(runs) if "/opt/trtmc-profile-downloader.py" in command + ) + prepare_index = next( + index + for index, command in enumerate(runs) + if "/src/.github/scripts/build-python-profiles.py" in command + ) + proof_index = next(index for index, command in enumerate(runs) if " --inner " in command) + assert download_index < prepare_index < proof_index + assert "--network bridge" in runs[download_index] + assert "--network none" in runs[prepare_index] + assert "--network none" in runs[proof_index] + assert f"dst={PREPARED_PROFILE_ROOT},readonly" in runs[proof_index] + assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1" in runs[proof_index] + + +def test_offline_proof_consumes_prepared_profiles_read_only() -> None: + source = RUNNER.read_text(encoding="utf-8") + proof = source.split("def _run_proof_container(", maxsplit=1)[1].split( + "def _proof_environment", maxsplit=1 + )[0] + environment = source.split("def _proof_environment(", maxsplit=1)[1].split( + "def _reclaim_orphans", maxsplit=1 + )[0] + + assert "dst={PREPARED_PROFILE_ROOT},\"" in proof + assert '"readonly"' in proof + assert '"--network"' in proof and '"none"' in proof + assert '"TRTMC_PYTHON_PROFILE_PREBUILT_ONLY": "1"' in environment + assert '"TRTMC_PYTHON_PROFILE_ROOT": PREPARED_PROFILE_ROOT' in environment + assert '"PIP_NO_INDEX": "1"' in environment + assert source.index("self._prepare_python_profiles(") < source.index("self.lease = GpuLease(") + + +def test_profile_download_program_fetches_a_digest_verified_sdist( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + artifact = b"source archive" + digest = hashlib.sha256(artifact).hexdigest() + metadata = json.dumps( + { + "urls": [ + { + "packagetype": "sdist", + "filename": "demo-1.0.0.tar.gz", + "url": "https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz", + "digests": {"sha256": digest}, + } + ] + } + ).encode() + + class Response(io.BytesIO): + def __init__(self, payload: bytes, url: str): + super().__init__(payload) + self.url = url + + def geturl(self) -> str: + return self.url + + def urlopen(request, timeout): + del timeout + url = str(request.full_url) + return Response( + metadata if url.endswith("/demo/1.0.0/json") else artifact, + url, + ) + + monkeypatch.setattr(profile_downloader.urllib.request, "urlopen", urlopen) + monkeypatch.setattr( + profile_downloader.subprocess, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=1), + ) + + profile_downloader.main([str(tmp_path), "demo==1.0.0"]) + + assert (tmp_path / "demo-1.0.0.tar.gz").read_bytes() == artifact + + +@pytest.mark.parametrize( + ("artifact_url", "expected_digest", "message"), + ( + ( + "https://example.invalid/demo-1.0.0.tar.gz", + hashlib.sha256(b"source archive").hexdigest(), + "untrusted source URL", + ), + ( + "https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz", + "0" * 64, + "digest mismatch", + ), + ), +) +def test_profile_download_program_rejects_untrusted_sdist_records( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + artifact_url: str, + expected_digest: str, + message: str, +) -> None: + artifact = b"source archive" + metadata = json.dumps( + { + "urls": [ + { + "packagetype": "sdist", + "filename": "demo-1.0.0.tar.gz", + "url": artifact_url, + "digests": {"sha256": expected_digest}, + } + ] + } + ).encode() + + class Response(io.BytesIO): + def __init__(self, payload: bytes, url: str): + super().__init__(payload) + self.url = url + + def geturl(self) -> str: + return self.url + + def urlopen(request, timeout): + del timeout + url = str(request.full_url) + return Response(metadata if url.endswith("/demo/1.0.0/json") else artifact, url) + + monkeypatch.setattr(profile_downloader.urllib.request, "urlopen", urlopen) + monkeypatch.setattr( + profile_downloader.subprocess, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=1), + ) + + with pytest.raises(RuntimeError, match=message): + profile_downloader.main([str(tmp_path), "demo==1.0.0"]) + + assert not (tmp_path / "demo-1.0.0.tar.gz").exists() + + +def test_projected_profile_packages_include_source_only_nemotron_dependencies( + tmp_path: Path, +) -> None: + projection = tmp_path / "projection" + package = projection / "python/tensorrt_model_connect" + family = package / "families/nemotron_h" + family.mkdir(parents=True) + shutil.copy2( + REPO_ROOT / "python/tensorrt_model_connect/families/nemotron_h/MODEL.toml", + family / "MODEL.toml", + ) + shutil.copytree( + REPO_ROOT + / "python/tensorrt_model_connect/families/nemotron_h/python_profile_requirements", + family / "python_profile_requirements", + ) + (package / "python_profiles.toml").write_text( + 'version = 1\n[profiles.base]\nkind = "passthrough"\n', + encoding="utf-8", + ) + runner = ModelProofRunner(CiContext(REPO_ROOT, {}), ModelProofRequest("nemotron_h")) + + packages = runner._projected_profile_packages(projection) + + assert "mamba-ssm==2.3.2.post1" in packages + assert "causal-conv1d==1.6.2.post1" in packages + + def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: artifacts = tmp_path / "artifacts" artifacts.mkdir() @@ -1781,6 +2091,15 @@ def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: "model-ci: error: unknown model \n", encoding="utf-8", ) + (artifacts / "python-profiles-prepare.log").write_text( + "profile install failed\n", + encoding="utf-8", + ) + (artifacts / "console.log").write_text( + "discarded-prefix\n" + ("x" * 20_000) + "\nbounded-tail\n", + encoding="utf-8", + ) + (artifacts / "build.log").symlink_to("/dev/zero") result = subprocess.run( [ @@ -1811,6 +2130,10 @@ def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: status = json.loads((artifacts / "model-proof-status.json").read_text(encoding="utf-8")) assert "host-error.log" in report assert "unknown model <unsafe>" in report + assert "python-profiles-prepare.log" in report + assert "profile install failed" in report + assert "bounded-tail" in report + assert "discarded-prefix" not in report assert status["outcome"] == "failed" assert status["exit_code"] == 2 diff --git a/tools/ci/README.md b/tools/ci/README.md index bfcebbcaba..16711e5bf9 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -158,12 +158,20 @@ The host half, `ModelProofRunner`, performs trusted setup: platform files, but no peer model source. 3. Select the model-owned runtime, Python tests, E2E cases, resource class, and optional reference checkout. -4. Prepare only the selected Hugging Face repositories and reflink them into a +4. In a restricted networked `runc` container, use the downloader embedded in + the reviewed base image to fetch only the selected family's exact-pinned + package artifacts. This step executes no contributor builder, verifier, or + package source code. +5. In a second `--network none`, no-GPU `runc` container, install those local + artifacts, execute the projected profile verifier, and publish the + proof-private virtual environments. +6. Prepare only the selected Hugging Face repositories and reflink them into a proof-private cache view. Premerge downloads a missing snapshot into the current host cache; nightly model proofs reuse the cache prepared by the separate Nightly cache-warm job. -5. Acquire either shared GPU slots or a whole GPU through `GpuLease`. -6. Start a read-only, network-disabled proof container. +7. Acquire either shared GPU slots or a whole GPU through `GpuLease`. +8. Start a read-only, network-disabled proof container with the prepared + profile directory mounted read-only. The container half, `ModelProofInnerPipeline`, then runs linearly: @@ -334,18 +342,15 @@ the producing class remains the source of truth for optional evidence fields. ### `docker_image.py` -- **Functionality / units:** `DockerImageManager` fingerprints image inputs, +- **Functionality / units:** `DockerImageManager` fingerprints stable base-image inputs, serializes concurrent builds with `WorkflowImageLock`, verifies dependency - versions, exposes Runtime Contract v2, and reuses only a matching local image. + versions, exposes Runtime Contract v3, and reuses only a matching local image. - **Inputs:** The common fingerprint includes the Dockerfile, `.dockerignore`, - profile builder and registry implementations, the normalized declarations of - every prebuilt Python profile, and their referenced lock and verification - files. The overlay fingerprint adds the exact TensorRT Python and APT - versions. Family metadata remains family-owned; comments, ownership fields, - lazy profiles, the general family loader, and package `__init__.py` metadata - are deliberately excluded because they do not change the baked environment. - The profile builder loads its narrow API through a synthetic package, so - package initialization is also absent from the actual image-build path. + and the model-agnostic PyPI artifact downloader copied into the image. The + overlay fingerprint adds the exact TensorRT Python and APT versions. + Family profile declarations, locks, verifiers, and preparation code are + deliberately excluded because their environments are not baked into the + base image. Contract-producer and CLI control-plane files are reviewed separately and do not perturb the semantic runtime fingerprint when their output is unchanged. - **Outputs:** Returns an immutable Docker ID shaped as @@ -354,13 +359,28 @@ the producing class remains the source of truth for optional evidence fields. `image_ref=sha256:...` through `GITHUB_OUTPUT`, and maintains a local verification stamp. `python3 -m tools.ci image contract` prints the complete canonical contract JSON, including the full common and overlay fingerprints - and `environment_contract_version=2`; callers may select an exact overlay + and `environment_contract_version=3`; callers may select an exact overlay with `--tensorrt-version` and `--tensorrt-apt-version`. - **Boundary:** It proves image identity and contents. It neither starts a container nor chooses a CI stage. Local image verification is serialized by a six-hour default lock budget, overridable with `TRTMC_CI_IMAGE_LOCK_TIMEOUT`. +### `profile_downloader.py` + +- **Functionality / units:** Downloads compatible wheels for exact public PyPI + pins and falls back to the release's unique source archive when no wheel is + available, without importing or building downloaded package code. +- **Inputs:** A proof-private destination plus validated `name==version` pins. + Network access is fixed to public PyPI and its package CDN; redirects and + source-archive SHA-256 digests are checked before publication. +- **Outputs:** A bounded directory of wheel and source-distribution artifacts + consumed later through `PIP_NO_INDEX=1` and `PIP_FIND_LINKS`. +- **Boundary:** This is the only online Python-profile operation. The downloader + is copied into and fingerprinted with the reviewed base image; installation, + source builds, and contributor verifiers run only in the separate offline + preparation container. + ### `container.py` - **Functionality / units:** `ContainerConfig` resolves run identity, workspace, @@ -634,26 +654,33 @@ the producing class remains the source of truth for optional evidence fields. ### `model_proof.py` -- **Functionality / units:** `ModelProofRunner` performs trusted host setup; +- **Functionality / units:** `ModelProofRunner` performs trusted host setup, + prepares projected family Python profiles in a restricted networked container; `ModelReferenceCache` first ensures the selected pinned checkout is present, then copies only that model-owned reference checkout; `ModelProofContainerCleaner` removes containers matching exact run labels. - **Inputs:** `ModelProofRequest {model, suite, revision, output_dir}`, full repository checkout, CI image, shared HF/reference cache roots, workflow identity, and model-proof GPU settings. -- **Outputs:** A positive `projection/`, proof-private `work/`, and +- **Outputs:** A positive `projection/`, proof-private `python-profiles/` when + the family declares profiles, `work/`, and `artifacts/` containing at least `selection.json`, `gpu-lease.json`, cache/reference evidence, `console.log`, `proof.json`, and `model-proof-report.html`. Host failures still attempt a fallback report. - **Boundary:** This is the trusted host/security boundary. It may read shared - caches and Docker state, but model build and inference occur only in the - network-disabled inner container. + caches and Docker state. The reviewed online profile downloader receives no + source, GPU, secrets, or shared cache and only fetches exact public PyPI artifacts. + Package installation, source builds, and the projected verifier run in a + separate network-disabled `runc` container with read-only source. Model build + and inference occur only in the network-disabled inner container, which + consumes the profile directory read-only. ### `model_proof_inner.py` - **Functionality / units:** `ModelProofInnerPipeline` runs the linear proof; `ProofStatus` records every phase so report generation can fail closed. -- **Inputs:** Read-only projected source at `/src`, writable `/work`, output +- **Inputs:** Read-only projected source at `/src`, prepared Python profiles + mounted read-only at `/opt/trtmc-python-profiles`, writable `/work`, output mount `/artifacts`, selected offline HF cache, optional private reference tree, one visible GPU, lease environment fields, and `ModelProofRequest`. - **Outputs:** Build/test/reference evidence plus these certification records: diff --git a/tools/ci/docker_image.py b/tools/ci/docker_image.py index 08c59287d0..0906ca0ad2 100644 --- a/tools/ci/docker_image.py +++ b/tools/ci/docker_image.py @@ -10,13 +10,10 @@ import fcntl import hashlib -import importlib.util import json import os import re -import sys import time -import types from dataclasses import dataclass from pathlib import Path @@ -24,7 +21,7 @@ FINGERPRINT_LABEL = "org.nvidia.trtmc.ci-input-fingerprint" -ENVIRONMENT_CONTRACT_VERSION = 2 +ENVIRONMENT_CONTRACT_VERSION = 3 IMMUTABLE_IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}") EXACT_TENSORRT_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+") EXACT_APT_VERSION = re.compile(r"[0-9][0-9A-Za-z.+:~_-]*") @@ -87,7 +84,6 @@ class ImageRequirements: tensorrt: str tensorrt_apt: str modelopt: str - python_profiles: str @property def python_distributions(self) -> dict[str, str]: @@ -109,7 +105,9 @@ def contract(self) -> dict[str, object]: "common_input_fingerprint": self.common_fingerprint, "input_fingerprint": self.fingerprint, "modelopt_version": self.modelopt, - "python_profiles": self.python_profiles.split(","), + # Kept as an empty compatibility field for runtime-catalog readers. + # Family-owned profiles are prepared per proof, not baked here. + "python_profiles": [], "tensorrt": { "version": self.tensorrt, "apt_version": self.tensorrt_apt, @@ -207,9 +205,8 @@ def ensure(self) -> str: print( f"CI Docker image '{image}' verified: TensorRT {versions['TENSORRT_VERSION']}, " f"exact Python, APT header, C++ header, and native runtime contracts, modelopt " - f"{versions['MODELOPT_VERSION']}, nlohmann/json headers, NeMo prompt RNN-T and " - f"prebuilt Python profiles ({versions['PYTHON_PROFILES']}) present, " - f"image {image_id}" + f"{versions['MODELOPT_VERSION']}, nlohmann/json headers, and NeMo prompt RNN-T " + f"present, image {image_id}" ) return image_id @@ -263,29 +260,10 @@ def _read_requirements( tensorrt_version: str | None = None, tensorrt_apt_version: str | None = None, ) -> ImageRequirements: - registry, profile_names = self._load_profile_registry() - profiles = registry["profiles"] - expected_profiles = ",".join(profile_names) - if not expected_profiles: - raise CiError("No prebuilt Python execution profiles were declared") - - package_root = Path("python/tensorrt_model_connect") - assets: set[Path] = set() - prebuilt_profiles = {name: profiles[name] for name in profile_names} - for spec in prebuilt_profiles.values(): - if not isinstance(spec, dict): - continue - for field in ("requirements", "verification_script_file"): - value = str(spec.get(field, "") or "").strip() - if value: - assets.add(self._profile_asset_input(package_root, value, field)) - inputs = { self.config.dockerfile, Path(".dockerignore"), - Path(".github/scripts/build-python-profiles.py"), - package_root / "python_profiles.py", - *assets, + Path("tools/ci/profile_downloader.py"), } dockerfile_text = (self.config.repository / self.config.dockerfile).read_text( encoding="utf-8" @@ -306,8 +284,6 @@ def _read_requirements( ) common_semantic_contract = { "environment_contract_version": ENVIRONMENT_CONTRACT_VERSION, - "version": registry.get("version"), - "profiles": prebuilt_profiles, } common_semantic_payload = json.dumps( common_semantic_contract, @@ -340,68 +316,11 @@ def _read_requirements( tensorrt, tensorrt_apt, modelopt, - expected_profiles, ) - def _profile_asset_input( - self, - package_root: Path, - path_spec: str, - field: str, - ) -> Path: - relative = Path(path_spec) - if relative.is_absolute() or ".." in relative.parts: - raise CiError(f"Python profile has an unsafe {field} path: {path_spec!r}") - absolute_root = (self.config.repository / package_root).resolve() - resolved = (absolute_root / relative).resolve() - try: - resolved.relative_to(absolute_root) - except ValueError as error: - raise CiError( - f"Python profile has an unsafe {field} path: {path_spec!r}" - ) from error - if not resolved.is_file(): - raise CiError( - f"Python profile references a missing {field} asset: {path_spec!r}" - ) - return package_root / relative - - def _load_profile_registry(self) -> tuple[dict[str, object], tuple[str, ...]]: - package_name = "tensorrt_model_connect" - package_root = self.config.repository / "python" / package_name - module_name = f"{package_name}.python_profiles" - previous_modules = { - name: module - for name, module in sys.modules.items() - if name == package_name or name.startswith(f"{package_name}.") - } - for name in previous_modules: - sys.modules.pop(name, None) - package = types.ModuleType(package_name) - package.__package__ = package_name - package.__path__ = [str(package_root)] - sys.modules[package_name] = package - try: - spec = importlib.util.spec_from_file_location( - module_name, - package_root / "python_profiles.py", - ) - if spec is None or spec.loader is None: - raise CiError("Could not load the Source Python profile registry") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - registry = module.load_python_profile_registry() - return registry, module.prebuilt_python_profile_names(registry) - finally: - for name in tuple(sys.modules): - if name == package_name or name.startswith(f"{package_name}."): - sys.modules.pop(name, None) - sys.modules.update(previous_modules) - def _fingerprint_inputs(self, inputs: tuple[Path, ...], semantic: str) -> str: digest = hashlib.sha256() - digest.update(b"python-profile-registry\0") + digest.update(b"ci-base-runtime\0") digest.update(semantic.encode("ascii") + b"\n") for relative in inputs: digest.update(str(relative).encode("utf-8") + b"\0") @@ -595,21 +514,6 @@ def _query_versions(self, image: str) -> dict[str, str]: print("MODELOPT_VERSION=" + metadata.version("nvidia-modelopt")) print("NLOHMANN_JSON_HEADER=" + ("present" if Path("/usr/include/nlohmann/json.hpp").is_file() else "missing")) print("NEMO_PROMPT_RNNT=available") -manifest_path = Path("/opt/trtmc-python-profiles/.image-ready.json") -manifest = json.loads(manifest_path.read_text(encoding="utf-8")) -if Path("/opt/trtmc-profile-source").exists(): - raise SystemExit("profile builder source leaked into the runtime image") -profiles = manifest.get("profiles") -if not isinstance(profiles, dict) or not profiles: - raise SystemExit("prebuilt Python profile manifest is empty or invalid") -for name, record in profiles.items(): - if not isinstance(record, dict): - raise SystemExit(f"invalid prebuilt Python profile record: {name}") - python = Path(str(record.get("python", ""))) - ready = Path(str(record.get("ready", ""))) - if not python.is_file() or not ready.is_file(): - raise SystemExit(f"prebuilt Python profile is incomplete: {name}") -print("PYTHON_PROFILES=" + ",".join(sorted(profiles))) """ result = self.commands.run( [ @@ -678,12 +582,6 @@ def _version_mismatches(actual: dict[str, str], expected: ImageRequirements) -> reasons.append("nlohmann/json development headers are missing") if actual.get("NEMO_PROMPT_RNNT") != "available": reasons.append("required NeMo prompt RNN-T capability is missing") - if actual.get("PYTHON_PROFILES") != expected.python_profiles: - reasons.append( - "prebuilt Python profiles differ: image has " - f"'{actual.get('PYTHON_PROFILES', 'missing')}', source expects " - f"'{expected.python_profiles}'" - ) return reasons def _build(self, image: str, expected: ImageRequirements, reasons: list[str]) -> None: diff --git a/tools/ci/model_proof.py b/tools/ci/model_proof.py index 54c1bedbfc..a522e0285f 100644 --- a/tools/ci/model_proof.py +++ b/tools/ci/model_proof.py @@ -18,6 +18,11 @@ from dataclasses import dataclass from pathlib import Path +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python 3.10 compatibility. + import tomli as tomllib + from .context import CiContext from .gpu_lease import GpuLease from .model_reference_cache import ModelReferenceCacheWarmer, ModelReferenceContract @@ -56,7 +61,19 @@ raise """ - +PREPARED_PROFILE_ROOT = "/opt/trtmc-python-profiles" +PROFILE_PACKAGES_ROOT = "/opt/trtmc-python-profile-packages" +_EXACT_PROFILE_REQUIREMENT_RE = re.compile( + r"^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[A-Za-z0-9,._-]+\])?==([^\s;]+)$" +) +_EXACT_PROFILE_VERSION_RE = re.compile( + r"(?:[0-9]+!)?[0-9]+(?:\.[0-9]+)*" + r"(?:(?:a|b|rc)[0-9]+)?" + r"(?:\.post[0-9]+)?" + r"(?:\.dev[0-9]+)?" + r"(?:\+[A-Za-z0-9]+(?:[._-][A-Za-z0-9]+)*)?", + re.IGNORECASE, +) @dataclass(frozen=True) class ModelProofRequest: """Validated command-line request for one projected model.""" @@ -274,8 +291,10 @@ def _run_host(self) -> None: projection = output / "projection" self.artifacts_dir = output / "artifacts" work = output / "work" + python_profiles = output / "python-profiles" + python_profile_packages = output / "python-profile-packages" output.mkdir(parents=True, exist_ok=True) - for path in (self.artifacts_dir, work): + for path in (self.artifacts_dir, work, python_profiles, python_profile_packages): if path.exists(): shutil.rmtree(path) path.mkdir(parents=True) @@ -305,6 +324,12 @@ def _run_host(self) -> None: ["docker", "image", "inspect", image], check=False, capture_output=True ).returncode: raise CiError(f"CI image is not present: {image}") + self._prepare_python_profiles( + projection, + python_profiles, + python_profile_packages, + image, + ) runtime_model = str(selection.payload["owners"]["runtime"]) validation_container = self._base_container_name() + "-validation-data" self.container_name = validation_container @@ -351,6 +376,7 @@ def _run_host(self) -> None: image, selection, validation_dir, + python_profiles, ) for name in ("proof.json", "model-proof-report.html"): if not (self.artifacts_dir / name).is_file(): @@ -393,6 +419,238 @@ def _project(self, projection: Path) -> None: if not (projection / ".trtmc-model-projection.json").is_file(): raise CiError("model_ci.py did not produce a projection manifest") + def _prepare_python_profiles( + self, + projection: Path, + profile_dir: Path, + package_dir: Path, + image: str, + ) -> None: + """Materialize projected profiles online for one later offline proof.""" + assert self.artifacts_dir is not None + packages = self._projected_profile_packages(projection) + if not packages: + return + self._download_python_profile_packages( + package_dir, + image, + packages, + ) + name = self._base_container_name() + "-python-profiles" + self.container_name = name + self.context.run(["docker", "rm", "-f", name], check=False, capture_output=True) + command = [ + "timeout", + "--kill-after=2m", + "6h", + "docker", + "run", + "--rm", + "--name", + name, + *self._job_labels(), + "--read-only", + "--network", + "none", + "--runtime", + "runc", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--ipc", + "private", + "--pids-limit", + "512", + "--memory", + "48g", + "--cpus", + "8", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--mount", + f"type=bind,src={projection},dst=/src,readonly", + "--mount", + f"type=bind,src={profile_dir},dst={PREPARED_PROFILE_ROOT}", + "--mount", + f"type=bind,src={package_dir},dst={PROFILE_PACKAGES_ROOT},readonly", + "--tmpfs", + "/tmp:rw,exec,nosuid,nodev,size=8g", + "--workdir", + "/src", + "-e", + "HOME=/tmp", + "-e", + "NVIDIA_VISIBLE_DEVICES=void", + "-e", + "CUDA_VISIBLE_DEVICES=", + "-e", + "PYTHONDONTWRITEBYTECODE=1", + "-e", + "PIP_DISABLE_PIP_VERSION_CHECK=1", + "-e", + "PIP_CONFIG_FILE=/dev/null", + "-e", + "PIP_FIND_LINKS=/opt/trtmc-python-profile-packages", + "-e", + "PIP_NO_CACHE_DIR=1", + "-e", + "PIP_NO_INDEX=1", + "-e", + "TRTMC_PYTHON_PROFILE_SOURCE=/src/python/tensorrt_model_connect", + "-e", + f"TRTMC_PYTHON_PROFILE_ROOT={PREPARED_PROFILE_ROOT}", + "-e", + "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=0", + image, + "/opt/venv/bin/python", + "/src/.github/scripts/build-python-profiles.py", + ] + result = self._run_logged( + command, + self.artifacts_dir / "python-profiles-prepare.log", + max_output_chars=2_000_000, + ) + if result: + raise CiError( + f"Python profile preparation failed for {self.request.model} (exit {result})" + ) + + def _projected_profile_packages(self, projection: Path) -> tuple[str, ...]: + """Return safe exact pins needed by the projected family's offline profiles.""" + package_root = projection / "python/tensorrt_model_connect" + manifests = sorted((package_root / "families").glob("*/MODEL.toml")) + if not manifests: + return () + family = tomllib.loads(manifests[0].read_text(encoding="utf-8")) + family_requirements: list[str] = [] + for raw_spec in family.get("python_profile_specs", []): + if not isinstance(raw_spec, str): + raise CiError("projected python_profile_specs must contain strings") + parts = [part.strip() for part in raw_spec.split("|")] + if len(parts) not in {3, 4, 5} or any(not part for part in parts[:3]): + raise CiError(f"invalid projected python_profile_specs entry: {raw_spec!r}") + prebuild = True + if len(parts) == 5: + value = parts[4].lower() + if value in {"0", "false", "no", "off"}: + prebuild = False + elif value not in {"1", "true", "yes", "on"}: + raise CiError(f"invalid projected profile prebuild flag: {parts[4]!r}") + if prebuild: + family_requirements.append(parts[1]) + if not family_requirements: + return () + + requirements = list(family_requirements) + registry_path = package_root / "python_profiles.toml" + registry = tomllib.loads(registry_path.read_text(encoding="utf-8")) + for profile, spec in registry.get("profiles", {}).items(): + if profile == "base" or not isinstance(spec, dict): + continue + if spec.get("kind") == "venv" and bool(spec.get("prebuild", True)): + value = spec.get("requirements") + if isinstance(value, str) and value.strip(): + requirements.append(value.strip()) + + result: set[str] = set() + for value in requirements: + relative = Path(value) + if relative.is_absolute() or ".." in relative.parts: + raise CiError(f"projected Python profile has an unsafe requirements path: {value!r}") + path = package_root / relative + if not path.is_file() or not path.resolve().is_relative_to(package_root.resolve()): + raise CiError(f"projected Python profile requirements are missing: {value!r}") + for line_number, raw_line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + match = _EXACT_PROFILE_REQUIREMENT_RE.fullmatch(line) + if match is None or _EXACT_PROFILE_VERSION_RE.fullmatch(match.group(2)) is None: + raise CiError( + f"projected Python profile lock must contain exact pins; " + f"{relative}:{line_number} is {raw_line!r}" + ) + result.add(line) + if len(result) > 256: + raise CiError("projected Python profiles declare more than 256 packages") + return tuple(sorted(result)) + + def _download_python_profile_packages( + self, + package_dir: Path, + image: str, + packages: tuple[str, ...], + ) -> None: + """Download wheels online without executing contributor-controlled package code.""" + assert self.artifacts_dir is not None + name = self._base_container_name() + "-python-profile-download" + self.container_name = name + self.context.run(["docker", "rm", "-f", name], check=False, capture_output=True) + command = [ + "timeout", + "--kill-after=1m", + "45m", + "docker", + "run", + "--rm", + "--name", + name, + *self._job_labels(), + "--read-only", + "--network", + "bridge", + "--runtime", + "runc", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--ipc", + "private", + "--pids-limit", + "128", + "--memory", + "4g", + "--cpus", + "2", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--mount", + f"type=bind,src={package_dir},dst={PROFILE_PACKAGES_ROOT}", + "--tmpfs", + "/tmp:rw,exec,nosuid,nodev,size=2g", + "--workdir", + "/tmp", + "-e", + "HOME=/tmp", + "-e", + "NVIDIA_VISIBLE_DEVICES=void", + "-e", + "CUDA_VISIBLE_DEVICES=", + "-e", + "PIP_CONFIG_FILE=/dev/null", + "-e", + "PIP_EXTRA_INDEX_URL=", + image, + "/opt/venv/bin/python", + "/opt/trtmc-profile-downloader.py", + PROFILE_PACKAGES_ROOT, + *packages, + ] + result = self._run_logged( + command, + self.artifacts_dir / "python-profile-download.log", + max_output_chars=2_000_000, + ) + if result: + raise CiError( + f"Python profile package download failed for {self.request.model} " + f"(exit {result})" + ) + def _job_labels(self) -> list[str]: return [ "--label", @@ -622,6 +880,7 @@ def _run_proof_container( image: str, selection: ModelProofSelection, validation_dir: Path | None, + python_profiles: Path, ) -> None: assert self.lease and self.artifacts_dir is not None and self.lease.gpu_id is not None name = self._base_container_name() @@ -638,6 +897,11 @@ def _run_proof_container( f"type=bind,src={self.artifacts_dir},dst=/artifacts", "--mount", f"type=bind,src={private_hub},dst=/hf-cache/hub", + "--mount", + ( + f"type=bind,src={python_profiles},dst={PREPARED_PROFILE_ROOT}," + "readonly" + ), ] if validation_dir is not None: mounts.extend( @@ -726,7 +990,10 @@ def _proof_environment( "TORCHINDUCTOR_CACHE_DIR": "/work/torch-cache", "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", + "PIP_NO_INDEX": "1", "PYTHONHASHSEED": "0", + "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY": "1", + "TRTMC_PYTHON_PROFILE_ROOT": PREPARED_PROFILE_ROOT, "TRTMC_MODEL_PLUGIN_STRICT": "1", "TRTMC_MODEL_PROOF_GPU_ID": str(self.lease.gpu_id), "TRTMC_MODEL_PROOF_GPU_SLOT_IDS": slots, @@ -811,7 +1078,13 @@ def _reclaim_orphans(self) -> None: if container in remaining: raise CiError(f"could not remove orphaned model-proof container {container}") - def _run_logged(self, command: list[object], path: Path) -> int: + def _run_logged( + self, + command: list[object], + path: Path, + *, + max_output_chars: int | None = None, + ) -> int: with path.open("w", encoding="utf-8") as output: process = subprocess.Popen( [str(item) for item in command], @@ -822,9 +1095,22 @@ def _run_logged(self, command: list[object], path: Path) -> int: stderr=subprocess.STDOUT, ) assert process.stdout is not None - for line in process.stdout: - print(line, end="") - output.write(line) + written = 0 + while chunk := process.stdout.read(8192): + if max_output_chars is not None and written + len(chunk) > max_output_chars: + remaining = max(0, max_output_chars - written) + if remaining: + print(chunk[:remaining], end="") + output.write(chunk[:remaining]) + message = "\nERROR: subprocess output limit exceeded\n" + print(message, end="") + output.write(message) + process.kill() + process.wait() + return 125 + print(chunk, end="") + output.write(chunk) + written += len(chunk) return process.wait() def _record_host_error(self, error: BaseException) -> None: diff --git a/tools/ci/profile_downloader.py b/tools/ci/profile_downloader.py new file mode 100644 index 0000000000..c5b9a65338 --- /dev/null +++ b/tools/ci/profile_downloader.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Download exact public PyPI artifacts without executing package source code. + +Boundary: online artifact retrieval only; profile installation and verification are offline. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import urllib.parse +import urllib.request + + +MAX_PACKAGE_COUNT = 256 +MAX_METADATA_BYTES = 4 * 1024 * 1024 +MAX_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024 +MAX_TOTAL_BYTES = 16 * 1024 * 1024 * 1024 + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _response_host(response, expected: str, label: str) -> None: + final_url = urllib.parse.urlparse(response.geturl()) + if final_url.scheme != "https" or final_url.hostname != expected: + raise RuntimeError(f"{label} redirected to an untrusted URL") + + +def _download_sdist(requirement: str, destination: Path) -> None: + distribution, separator, version = requirement.partition("==") + if not separator: + raise RuntimeError(f"invalid exact requirement: {requirement}") + distribution = distribution.partition("[")[0] + endpoint = ( + "https://pypi.org/pypi/" + + urllib.parse.quote(distribution, safe="") + + "/" + + urllib.parse.quote(version, safe="") + + "/json" + ) + request = urllib.request.Request( + endpoint, + headers={"User-Agent": "trtmc-profile-preparer/1"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + _response_host(response, "pypi.org", f"PyPI metadata for {requirement}") + payload_bytes = response.read(MAX_METADATA_BYTES + 1) + if len(payload_bytes) > MAX_METADATA_BYTES: + raise RuntimeError(f"PyPI metadata is unexpectedly large for {requirement}") + payload = json.loads(payload_bytes) + candidates = [ + item for item in payload.get("urls", []) if item.get("packagetype") == "sdist" + ] + if len(candidates) != 1: + raise RuntimeError(f"no unique source distribution is available for {requirement}") + record = candidates[0] + filename = record.get("filename") + url = record.get("url") + expected = record.get("digests", {}).get("sha256") + if ( + not isinstance(filename, str) + or Path(filename).name != filename + or not isinstance(url, str) + or not isinstance(expected, str) + or len(expected) != 64 + ): + raise RuntimeError(f"PyPI returned an invalid source record for {requirement}") + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https" or parsed.hostname != "files.pythonhosted.org": + raise RuntimeError(f"PyPI returned an untrusted source URL for {requirement}") + target = destination / filename + if target.is_file() and _sha256(target) == expected: + return + temporary = destination / f".{filename}.{os.getpid()}.tmp" + artifact_request = urllib.request.Request( + url, + headers={"User-Agent": "trtmc-profile-preparer/1"}, + ) + digest = hashlib.sha256() + size = 0 + try: + with urllib.request.urlopen(artifact_request, timeout=300) as response, temporary.open( + "wb" + ) as output: + _response_host(response, "files.pythonhosted.org", requirement) + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + size += len(chunk) + if size > MAX_ARTIFACT_BYTES: + raise RuntimeError(f"source distribution is too large for {requirement}") + digest.update(chunk) + output.write(chunk) + if digest.hexdigest() != expected: + raise RuntimeError(f"source distribution digest mismatch for {requirement}") + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + + +def download_packages(destination: Path, requirements: list[str]) -> None: + if not requirements or len(requirements) > MAX_PACKAGE_COUNT: + raise ValueError(f"profile package count must be between 1 and {MAX_PACKAGE_COUNT}") + destination.mkdir(parents=True, exist_ok=True) + for requirement in requirements: + result = subprocess.run( + [ + sys.executable, + "-m", + "pip", + "download", + "--disable-pip-version-check", + "--no-cache-dir", + "--no-deps", + "--only-binary=:all:", + "--index-url", + "https://pypi.org/simple", + "--dest", + str(destination), + requirement, + ], + check=False, + ) + if result.returncode: + _download_sdist(requirement, destination) + total_bytes = sum( + path.stat().st_size + for path in destination.iterdir() + if path.is_file() and not path.is_symlink() + ) + if total_bytes > MAX_TOTAL_BYTES: + raise RuntimeError("prepared profile packages exceed the 16 GiB run limit") + + +def main(arguments: list[str] | None = None) -> int: + values = list(sys.argv[1:] if arguments is None else arguments) + if len(values) < 2: + raise SystemExit("usage: profile_downloader.py DESTINATION NAME==VERSION [...]") + download_packages(Path(values[0]), values[1:]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/model_ci.py b/tools/model_ci.py index bfb8e95eef..8c0b57ec4d 100644 --- a/tools/model_ci.py +++ b/tools/model_ci.py @@ -49,6 +49,7 @@ PLATFORM_PROJECTION_EXACT = frozenset( { ".clang-format", + ".github/scripts/build-python-profiles.py", ".github/scripts/write-model-proof-fallback-report.py", "ASSET_LICENSES.md", "CMakeLists.txt", diff --git a/website/docs/extend/add-model-family.md b/website/docs/extend/add-model-family.md index e0b9d7f564..4d6e789b93 100644 --- a/website/docs/extend/add-model-family.md +++ b/website/docs/extend/add-model-family.md @@ -103,6 +103,41 @@ policy, and optional debug hooks in this family package. The old repository-root `graph_ops.py`, `graph_blocks.py`, and `standard_decoder_builder.py` ownership model has been retired. +### Optional Python execution profile + +If build or reference code needs Python packages that conflict with the common +environment, keep the declaration and exact pins in the owning family: + +```toml +python_profile_specs = [ + "example_reference|families/example/python_profile_requirements/reference.lock.txt|families/example/python_profile_verify.py|true|true", +] +default_execution_profiles = [ + "reference|example_reference", +] +``` + +The fields are `name|requirements|verifier|system_site_packages|prebuild`. +`prebuild=true` means CI prepares the profile before a network-disabled proof; +it does not bake the profile into the shared base image or change the base +runtime fingerprint. Requirements must be exact public PyPI `name==version` +pins. CI downloads their artifacts with the reviewed base-image downloader, +then installs, source-builds, and verifies them in a separate offline container. + +When an sdist must disable its own network-wheel lookup, declare only the +package build setting in the family descriptor: + +```toml +python_profile_build_environment = [ + "example_reference|PACKAGE_FORCE_BUILD|TRUE", +] +``` + +This surface is for package build switches, not credentials, package indexes, +runtime configuration, or system dependencies. A new APT, CUDA, compiler, or +system-library requirement still changes the reviewed base runtime and needs +maintainer qualification. + ### Optional split decoder contract Opt into separate prefill/decode engines only when the family builder and From b2e60b043063fda688e027766aff7a5dd5cd1817 Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:38:48 -0700 Subject: [PATCH 2/7] test(ci): cover prepared profile mount Update the selected-wheel proof test for the new prepared-profile argument and assert the read-only mount plus offline profile contract. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- tests/tools/test_selected_wheel_runtime.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/tools/test_selected_wheel_runtime.py b/tests/tools/test_selected_wheel_runtime.py index ca6a5ba3ca..bc78ef97ec 100644 --- a/tests/tools/test_selected_wheel_runtime.py +++ b/tests/tools/test_selected_wheel_runtime.py @@ -211,6 +211,8 @@ def test_model_proof_mounts_selected_wheels_read_only_and_forwards_contract( runner.artifacts_dir = tmp_path / "artifacts" runner.artifacts_dir.mkdir() runner.revision = "a" * 40 + python_profiles = tmp_path / "python-profiles" + python_profiles.mkdir() captured: list[list[object]] = [] monkeypatch.setattr( context, @@ -230,9 +232,16 @@ def test_model_proof_mounts_selected_wheels_read_only_and_forwards_contract( "fixture-image", SimpleNamespace(reference_cache=None), None, + python_profiles, ) command = captured[0] + assert ( + f"type=bind,src={python_profiles},dst=/opt/trtmc-python-profiles,readonly" + in command + ) + assert "TRTMC_PYTHON_PROFILE_ROOT=/opt/trtmc-python-profiles" in command + assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1" in command assert f"type=bind,src={selected.resolve()},dst=/selected-wheel,readonly" in command assert "TRTMC_SELECTED_WHEEL_DIR=/selected-wheel" in command assert f"TRTMC_SELECTED_WHEEL_PYTHON_TAG={PYTHON_TAG}" in command From 8b9a282a3e23fce58ccf57d84a9da1ecb93bb1c6 Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:57:28 -0700 Subject: [PATCH 3/7] fix(ci): harden profile proof boundaries Open fallback diagnostics without blocking so non-regular files cannot stall report generation. Keep proof command checks tied to the actual projection and package mount constants. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- .github/scripts/write-model-proof-fallback-report.py | 7 ++++++- tests/tools/test_model_proof_runner.py | 8 +++++--- tools/ci/model_proof.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/scripts/write-model-proof-fallback-report.py b/.github/scripts/write-model-proof-fallback-report.py index 49d7720d2a..f0a063b6c9 100644 --- a/.github/scripts/write-model-proof-fallback-report.py +++ b/.github/scripts/write-model-proof-fallback-report.py @@ -43,7 +43,12 @@ def _diagnostics(root: Path) -> list[tuple[str, str]]: for filename in _DIAGNOSTIC_FILES: path = root / filename try: - descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) except OSError: continue try: diff --git a/tests/tools/test_model_proof_runner.py b/tests/tools/test_model_proof_runner.py index 7d7d65f5fd..44a7bdfdda 100644 --- a/tests/tools/test_model_proof_runner.py +++ b/tests/tools/test_model_proof_runner.py @@ -1872,7 +1872,7 @@ def prepare(command: list[object], _log: Path, **_kwargs) -> int: assert "/opt/trtmc-profile-downloader.py" in download assert "demo-package==1.0.0" in download assert f"src={packages},dst={PROFILE_PACKAGES_ROOT}" in download - assert "src={projection}" not in download + assert f"src={projection}" not in download assert "NVIDIA_VISIBLE_DEVICES=void" in download assert "CUDA_VISIBLE_DEVICES=" in download @@ -1883,7 +1883,7 @@ def prepare(command: list[object], _log: Path, **_kwargs) -> int: assert "--security-opt no-new-privileges" in install assert "--ipc private" in install assert "PIP_CONFIG_FILE=/dev/null" in install - assert "PIP_FIND_LINKS=/opt/trtmc-python-profile-packages" in install + assert f"PIP_FIND_LINKS={PROFILE_PACKAGES_ROOT}" in install assert "PIP_NO_INDEX=1" in install assert f"src={projection},dst=/src,readonly" in install assert f"src={profiles},dst={PREPARED_PROFILE_ROOT}" in install @@ -2099,7 +2099,8 @@ def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: "discarded-prefix\n" + ("x" * 20_000) + "\nbounded-tail\n", encoding="utf-8", ) - (artifacts / "build.log").symlink_to("/dev/zero") + (artifacts / "configure.log").symlink_to("/dev/zero") + os.mkfifo(artifacts / "build.log") result = subprocess.run( [ @@ -2123,6 +2124,7 @@ def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: capture_output=True, text=True, check=False, + timeout=5, ) assert result.returncode == 0, result.stderr diff --git a/tools/ci/model_proof.py b/tools/ci/model_proof.py index a522e0285f..d0baa2b0c8 100644 --- a/tools/ci/model_proof.py +++ b/tools/ci/model_proof.py @@ -491,7 +491,7 @@ def _prepare_python_profiles( "-e", "PIP_CONFIG_FILE=/dev/null", "-e", - "PIP_FIND_LINKS=/opt/trtmc-python-profile-packages", + f"PIP_FIND_LINKS={PROFILE_PACKAGES_ROOT}", "-e", "PIP_NO_CACHE_DIR=1", "-e", From a743a4135ca7c93db003298e1c0d39837296c402 Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:10:07 -0700 Subject: [PATCH 4/7] fix(ci): prepare empty Python profiles Distinguish a projection with no prebuilt profile from one whose declared profile needs no downloaded packages. Always run offline materialization for the latter. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- tests/tools/test_model_proof_runner.py | 61 ++++++++++++++++++++++++++ tools/ci/model_proof.py | 21 ++++----- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/tests/tools/test_model_proof_runner.py b/tests/tools/test_model_proof_runner.py index 44a7bdfdda..c501fe9f5d 100644 --- a/tests/tools/test_model_proof_runner.py +++ b/tests/tools/test_model_proof_runner.py @@ -1929,6 +1929,67 @@ def test_profile_owning_family_runs_download_prepare_then_offline_proof( assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1" in runs[proof_index] +def test_empty_profile_lock_still_runs_offline_preparation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + projection = tmp_path / "projection" + family = projection / "python/tensorrt_model_connect/families/demo" + family.mkdir(parents=True) + (family / "MODEL.toml").write_text( + 'id = "demo"\n' + 'python_profile_specs = [' + '"demo|families/demo/requirements.lock.txt|families/demo/verify.py|true"' + "]\n", + encoding="utf-8", + ) + (family / "requirements.lock.txt").write_text( + "# no additional packages\n", + encoding="utf-8", + ) + (family / "verify.py").write_text("assert True\n", encoding="utf-8") + package_root = projection / "python/tensorrt_model_connect" + (package_root / "python_profiles.toml").write_text( + 'version = 1\n[profiles.base]\nkind = "passthrough"\n', + encoding="utf-8", + ) + profiles = tmp_path / "python-profiles" + profiles.mkdir() + packages = tmp_path / "python-profile-packages" + packages.mkdir() + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + runner = ModelProofRunner( + CiContext(REPO_ROOT, {}), + ModelProofRequest(model="demo"), + ) + runner.artifacts_dir = artifacts + monkeypatch.setattr( + runner.context, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=0), + ) + commands: list[list[object]] = [] + monkeypatch.setattr( + runner, + "_run_logged", + lambda command, _log, **_kwargs: commands.append(command) or 0, + ) + + runner._prepare_python_profiles( + projection, + profiles, + packages, + "qualified-base@sha256:test", + ) + + assert len(commands) == 1 + preparation = " ".join(map(str, commands[0])) + assert "--network none" in preparation + assert "/src/.github/scripts/build-python-profiles.py" in preparation + assert "/opt/trtmc-profile-downloader.py" not in preparation + + def test_offline_proof_consumes_prepared_profiles_read_only() -> None: source = RUNNER.read_text(encoding="utf-8") proof = source.split("def _run_proof_container(", maxsplit=1)[1].split( diff --git a/tools/ci/model_proof.py b/tools/ci/model_proof.py index d0baa2b0c8..c60bb5e6be 100644 --- a/tools/ci/model_proof.py +++ b/tools/ci/model_proof.py @@ -429,13 +429,14 @@ def _prepare_python_profiles( """Materialize projected profiles online for one later offline proof.""" assert self.artifacts_dir is not None packages = self._projected_profile_packages(projection) - if not packages: + if packages is None: return - self._download_python_profile_packages( - package_dir, - image, - packages, - ) + if packages: + self._download_python_profile_packages( + package_dir, + image, + packages, + ) name = self._base_container_name() + "-python-profiles" self.container_name = name self.context.run(["docker", "rm", "-f", name], check=False, capture_output=True) @@ -516,12 +517,12 @@ def _prepare_python_profiles( f"Python profile preparation failed for {self.request.model} (exit {result})" ) - def _projected_profile_packages(self, projection: Path) -> tuple[str, ...]: - """Return safe exact pins needed by the projected family's offline profiles.""" + def _projected_profile_packages(self, projection: Path) -> tuple[str, ...] | None: + """Return exact pins, or None when the projection has no prebuilt profile.""" package_root = projection / "python/tensorrt_model_connect" manifests = sorted((package_root / "families").glob("*/MODEL.toml")) if not manifests: - return () + return None family = tomllib.loads(manifests[0].read_text(encoding="utf-8")) family_requirements: list[str] = [] for raw_spec in family.get("python_profile_specs", []): @@ -540,7 +541,7 @@ def _projected_profile_packages(self, projection: Path) -> tuple[str, ...]: if prebuild: family_requirements.append(parts[1]) if not family_requirements: - return () + return None requirements = list(family_requirements) registry_path = package_root / "python_profiles.toml" From aa0fe8bb6d0fc761ba119910260b8a5448d08ec1 Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:24:18 -0700 Subject: [PATCH 5/7] fix(ci): prepare selected generic profiles Recognize generic profiles selected by family defaults, E2E overrides, and strategy defaults before offline proof. Leave unselected generic profiles out of ordinary model preparation. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- tests/tools/test_model_proof_runner.py | 51 ++++++++++----- tools/ci/model_proof.py | 87 ++++++++++++++++++++++++-- 2 files changed, 117 insertions(+), 21 deletions(-) diff --git a/tests/tools/test_model_proof_runner.py b/tests/tools/test_model_proof_runner.py index c501fe9f5d..68a0b4d9c6 100644 --- a/tests/tools/test_model_proof_runner.py +++ b/tests/tools/test_model_proof_runner.py @@ -1929,30 +1929,49 @@ def test_profile_owning_family_runs_download_prepare_then_offline_proof( assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1" in runs[proof_index] +@pytest.mark.parametrize("profile_owner", ("family", "generic")) def test_empty_profile_lock_still_runs_offline_preparation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + profile_owner: str, ) -> None: projection = tmp_path / "projection" family = projection / "python/tensorrt_model_connect/families/demo" family.mkdir(parents=True) - (family / "MODEL.toml").write_text( - 'id = "demo"\n' - 'python_profile_specs = [' - '"demo|families/demo/requirements.lock.txt|families/demo/verify.py|true"' - "]\n", - encoding="utf-8", - ) - (family / "requirements.lock.txt").write_text( - "# no additional packages\n", - encoding="utf-8", - ) - (family / "verify.py").write_text("assert True\n", encoding="utf-8") package_root = projection / "python/tensorrt_model_connect" - (package_root / "python_profiles.toml").write_text( - 'version = 1\n[profiles.base]\nkind = "passthrough"\n', - encoding="utf-8", - ) + if profile_owner == "family": + (family / "MODEL.toml").write_text( + 'id = "demo"\n' + 'python_profile_specs = [' + '"demo|families/demo/requirements.lock.txt|families/demo/verify.py|true"' + "]\n", + encoding="utf-8", + ) + (family / "requirements.lock.txt").write_text( + "# no additional packages\n", + encoding="utf-8", + ) + (family / "verify.py").write_text("assert True\n", encoding="utf-8") + registry = 'version = 1\n[profiles.base]\nkind = "passthrough"\n' + else: + (family / "MODEL.toml").write_text( + 'id = "demo"\n' + 'default_execution_profiles = ["reference|generic"]\n', + encoding="utf-8", + ) + requirements = package_root / "python_profile_requirements" + requirements.mkdir() + (requirements / "generic.lock.txt").write_text( + "# no additional packages\n", + encoding="utf-8", + ) + registry = ( + 'version = 1\n[profiles.base]\nkind = "passthrough"\n' + '[profiles.generic]\nkind = "venv"\n' + 'requirements = "python_profile_requirements/generic.lock.txt"\n' + 'verification_script = "assert True"\n' + ) + (package_root / "python_profiles.toml").write_text(registry, encoding="utf-8") profiles = tmp_path / "python-profiles" profiles.mkdir() packages = tmp_path / "python-profile-packages" diff --git a/tools/ci/model_proof.py b/tools/ci/model_proof.py index c60bb5e6be..85735cea29 100644 --- a/tools/ci/model_proof.py +++ b/tools/ci/model_proof.py @@ -540,19 +540,33 @@ def _projected_profile_packages(self, projection: Path) -> tuple[str, ...] | Non raise CiError(f"invalid projected profile prebuild flag: {parts[4]!r}") if prebuild: family_requirements.append(parts[1]) - if not family_requirements: - return None - requirements = list(family_requirements) registry_path = package_root / "python_profiles.toml" registry = tomllib.loads(registry_path.read_text(encoding="utf-8")) + selected_profiles = self._projected_selected_profile_names( + projection, + family, + registry, + ) + generic_requirements: list[tuple[str, str]] = [] for profile, spec in registry.get("profiles", {}).items(): if profile == "base" or not isinstance(spec, dict): continue if spec.get("kind") == "venv" and bool(spec.get("prebuild", True)): value = spec.get("requirements") - if isinstance(value, str) and value.strip(): - requirements.append(value.strip()) + if not isinstance(value, str) or not value.strip(): + raise CiError( + f"projected Python profile {profile!r} has no requirements" + ) + generic_requirements.append((str(profile), value.strip())) + if not family_requirements and not any( + profile in selected_profiles for profile, _requirements in generic_requirements + ): + return None + requirements = [ + *family_requirements, + *(value for _profile, value in generic_requirements), + ] result: set[str] = set() for value in requirements: @@ -579,6 +593,69 @@ def _projected_profile_packages(self, projection: Path) -> tuple[str, ...] | Non raise CiError("projected Python profiles declare more than 256 packages") return tuple(sorted(result)) + @staticmethod + def _projected_selected_profile_names( + projection: Path, + family: dict[str, object], + registry: dict[str, object], + ) -> set[str]: + """Return profiles selected by projected family and E2E metadata.""" + selected: set[str] = set() + + raw_defaults = family.get("default_execution_profiles", []) + if not isinstance(raw_defaults, list): + raise CiError("projected default_execution_profiles must be a list") + for raw_default in raw_defaults: + if not isinstance(raw_default, str): + raise CiError("projected default_execution_profiles must contain strings") + parts = [part.strip() for part in raw_default.split("|")] + if len(parts) != 2 or any(not part for part in parts): + raise CiError( + f"invalid projected default_execution_profiles entry: {raw_default!r}" + ) + selected.add(parts[1]) + + def add_profiles(value: object, label: str) -> None: + if value is None: + return + if not isinstance(value, dict): + raise CiError(f"projected {label} must be an object") + for profile in value.values(): + if not isinstance(profile, str) or not profile.strip(): + raise CiError(f"projected {label} must select non-empty profiles") + selected.add(profile.strip()) + + e2e_root = projection / "tests/e2e/models" + for manifest_path in sorted(e2e_root.glob("*/manifests/*.json")): + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise CiError(f"projected E2E manifest must be an object: {manifest_path}") + testcases = manifest.get("testcases", []) + if not isinstance(testcases, list): + raise CiError(f"projected E2E testcases must be a list: {manifest_path}") + cases = [manifest] + for testcase in testcases: + if not isinstance(testcase, dict): + raise CiError(f"projected E2E testcase must be an object: {manifest_path}") + cases.append({**manifest, **testcase}) + for case in cases: + add_profiles(case.get("execution_profiles"), "execution_profiles") + for section_name, selector_field in ( + ("runtime_strategy_defaults", "runtime_strategy"), + ("reference_backend_defaults", "reference_backend"), + ): + selector = case.get(selector_field) + if not isinstance(selector, str) or not selector.strip(): + continue + section = registry.get(section_name, {}) + if not isinstance(section, dict): + raise CiError(f"projected {section_name} must be an object") + add_profiles( + section.get(selector.strip()), + f"{section_name}.{selector.strip()}", + ) + return selected + def _download_python_profile_packages( self, package_dir: Path, From 0d4eec09bba39d436c104c0f84ba9cf1352d7f67 Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:41:16 -0700 Subject: [PATCH 6/7] fix(ci): isolate selected profile preparation Build an exact profile-and-package plan from family and E2E defaults. Pass the selected names to the offline builder so unrelated generic profiles cannot affect a model proof. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- .github/scripts/build-python-profiles.py | 20 +++++- tests/tools/test_ensure_ci_docker_image.py | 42 ++++++++++++ tests/tools/test_model_proof_runner.py | 55 ++++++++++++++- tools/ci/model_proof.py | 80 ++++++++++++++++------ 4 files changed, 170 insertions(+), 27 deletions(-) diff --git a/.github/scripts/build-python-profiles.py b/.github/scripts/build-python-profiles.py index c2b15ab3db..0623ee4595 100644 --- a/.github/scripts/build-python-profiles.py +++ b/.github/scripts/build-python-profiles.py @@ -5,6 +5,7 @@ from __future__ import annotations +import argparse import importlib.util import os import sys @@ -38,13 +39,26 @@ def _load_profile_api(): return module -def main() -> None: +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--profile", action="append", default=[]) + args = parser.parse_args(argv) profile_api = _load_profile_api() - names = profile_api.prebuilt_python_profile_names( + available = profile_api.prebuilt_python_profile_names( profile_api.load_python_profile_registry() ) - if not names: + if not available: raise SystemExit("no prebuilt Python profiles were declared") + requested = tuple(args.profile) + if requested: + if len(set(requested)) != len(requested): + raise SystemExit("requested Python profiles must be unique") + unknown = sorted(set(requested) - set(available)) + if unknown: + raise SystemExit("requested Python profiles are not prebuilt: " + ",".join(unknown)) + names = tuple(sorted(requested)) + else: + names = available base_python = os.environ.get("TRTMC_BASE_PYTHON", "/opt/venv/bin/python") for name in names: diff --git a/tests/tools/test_ensure_ci_docker_image.py b/tests/tools/test_ensure_ci_docker_image.py index 4f559fb646..68d9762bd8 100644 --- a/tests/tools/test_ensure_ci_docker_image.py +++ b/tests/tools/test_ensure_ci_docker_image.py @@ -814,6 +814,48 @@ def test_profile_builder_does_not_execute_package_init(tmp_path: Path) -> None: assert "package init must not execute" not in result.stderr +def test_profile_builder_materializes_only_requested_profiles(tmp_path: Path) -> None: + package_root = tmp_path / "tensorrt_model_connect" + package_root.mkdir() + profile_pythons: dict[str, Path] = {} + for name in ("selected", "unselected"): + profile = tmp_path / name + (profile / "bin").mkdir(parents=True) + (profile / ".ready").touch() + profile_pythons[name] = profile / "bin/python" + serialized_pythons = {name: str(path) for name, path in profile_pythons.items()} + (package_root / "python_profiles.py").write_text( + "def load_python_profile_registry(): return {}\n" + "def prebuilt_python_profile_names(registry): " + "return ('selected', 'unselected')\n" + f"PROFILE_PYTHONS = {serialized_pythons!r}\n" + "def resolve_profile_python(name, base_python):\n" + " print('resolved_profile=' + name)\n" + " return str(PROFILE_PYTHONS[name])\n", + encoding="utf-8", + ) + env = os.environ.copy() + env["TRTMC_PYTHON_PROFILE_SOURCE"] = str(package_root) + + result = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / ".github/scripts/build-python-profiles.py"), + "--profile", + "selected", + ], + env=env, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "resolved_profile=selected" in result.stdout + assert "resolved_profile=unselected" not in result.stdout + assert "prepared_python_profiles=selected" in result.stdout + + def test_source_contract_does_not_execute_or_fingerprint_family_loader( tmp_path: Path, ) -> None: diff --git a/tests/tools/test_model_proof_runner.py b/tests/tools/test_model_proof_runner.py index 68a0b4d9c6..5ca3a16ef8 100644 --- a/tests/tools/test_model_proof_runner.py +++ b/tests/tools/test_model_proof_runner.py @@ -1965,11 +1965,18 @@ def test_empty_profile_lock_still_runs_offline_preparation( "# no additional packages\n", encoding="utf-8", ) + (requirements / "unselected.lock.txt").write_text( + "unselected-package==1.0.0\n", + encoding="utf-8", + ) registry = ( 'version = 1\n[profiles.base]\nkind = "passthrough"\n' '[profiles.generic]\nkind = "venv"\n' 'requirements = "python_profile_requirements/generic.lock.txt"\n' 'verification_script = "assert True"\n' + '[profiles.unselected]\nkind = "venv"\n' + 'requirements = "python_profile_requirements/unselected.lock.txt"\n' + 'verification_script = "assert True"\n' ) (package_root / "python_profiles.toml").write_text(registry, encoding="utf-8") profiles = tmp_path / "python-profiles" @@ -2007,6 +2014,47 @@ def test_empty_profile_lock_still_runs_offline_preparation( assert "--network none" in preparation assert "/src/.github/scripts/build-python-profiles.py" in preparation assert "/opt/trtmc-profile-downloader.py" not in preparation + assert f"--profile {'demo' if profile_owner == 'family' else 'generic'}" in preparation + + +def test_e2e_defaults_select_a_generic_profile_for_preparation(tmp_path: Path) -> None: + projection = tmp_path / "projection" + package_root = projection / "python/tensorrt_model_connect" + requirements = package_root / "python_profile_requirements" + requirements.mkdir(parents=True) + (requirements / "generic.lock.txt").write_text( + "generic-package==1.2.3\n", + encoding="utf-8", + ) + (package_root / "python_profiles.toml").write_text( + 'version = 1\n[profiles.base]\nkind = "passthrough"\n' + '[profiles.generic]\nkind = "venv"\n' + 'requirements = "python_profile_requirements/generic.lock.txt"\n' + 'verification_script = "assert True"\n' + '[reference_backend_defaults.hf_transformers]\nreference = "generic"\n', + encoding="utf-8", + ) + e2e = projection / "tests/e2e/models/demo" + (e2e / "manifests").mkdir(parents=True) + (e2e / "MODEL.toml").write_text( + '[e2e_defaults.demo_task]\nreference_backend = "hf_transformers"\n', + encoding="utf-8", + ) + (e2e / "manifests/demo.json").write_text( + json.dumps({ + "name": "demo", + "task_strategy": "demo_task", + "testcases": [{"name": "demo"}], + }), + encoding="utf-8", + ) + runner = ModelProofRunner(CiContext(REPO_ROOT, {}), ModelProofRequest("demo")) + + plan = runner._projected_python_profile_plan(projection) + + assert plan is not None + assert plan.names == ("generic",) + assert plan.packages == ("generic-package==1.2.3",) def test_offline_proof_consumes_prepared_profiles_read_only() -> None: @@ -2158,10 +2206,11 @@ def test_projected_profile_packages_include_source_only_nemotron_dependencies( ) runner = ModelProofRunner(CiContext(REPO_ROOT, {}), ModelProofRequest("nemotron_h")) - packages = runner._projected_profile_packages(projection) + plan = runner._projected_python_profile_plan(projection) - assert "mamba-ssm==2.3.2.post1" in packages - assert "causal-conv1d==1.6.2.post1" in packages + assert plan is not None + assert "mamba-ssm==2.3.2.post1" in plan.packages + assert "causal-conv1d==1.6.2.post1" in plan.packages def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: diff --git a/tools/ci/model_proof.py b/tools/ci/model_proof.py index 85735cea29..d4dc14c19b 100644 --- a/tools/ci/model_proof.py +++ b/tools/ci/model_proof.py @@ -74,6 +74,16 @@ r"(?:\+[A-Za-z0-9]+(?:[._-][A-Za-z0-9]+)*)?", re.IGNORECASE, ) + + +@dataclass(frozen=True) +class _PythonProfilePlan: + """Exact projected profiles and packages prepared before offline proof.""" + + names: tuple[str, ...] + packages: tuple[str, ...] + + @dataclass(frozen=True) class ModelProofRequest: """Validated command-line request for one projected model.""" @@ -428,14 +438,14 @@ def _prepare_python_profiles( ) -> None: """Materialize projected profiles online for one later offline proof.""" assert self.artifacts_dir is not None - packages = self._projected_profile_packages(projection) - if packages is None: + plan = self._projected_python_profile_plan(projection) + if plan is None: return - if packages: + if plan.packages: self._download_python_profile_packages( package_dir, image, - packages, + plan.packages, ) name = self._base_container_name() + "-python-profiles" self.container_name = name @@ -506,6 +516,7 @@ def _prepare_python_profiles( image, "/opt/venv/bin/python", "/src/.github/scripts/build-python-profiles.py", + *(item for name in plan.names for item in ("--profile", name)), ] result = self._run_logged( command, @@ -517,14 +528,21 @@ def _prepare_python_profiles( f"Python profile preparation failed for {self.request.model} (exit {result})" ) - def _projected_profile_packages(self, projection: Path) -> tuple[str, ...] | None: - """Return exact pins, or None when the projection has no prebuilt profile.""" + def _projected_python_profile_plan( + self, + projection: Path, + ) -> _PythonProfilePlan | None: + """Return selected profiles and pins, or None when none need preparation.""" package_root = projection / "python/tensorrt_model_connect" manifests = sorted((package_root / "families").glob("*/MODEL.toml")) - if not manifests: - return None - family = tomllib.loads(manifests[0].read_text(encoding="utf-8")) - family_requirements: list[str] = [] + if len(manifests) > 1: + raise CiError("projected Python ownership contains multiple families") + family = ( + tomllib.loads(manifests[0].read_text(encoding="utf-8")) + if manifests + else {} + ) + family_profiles: list[tuple[str, str]] = [] for raw_spec in family.get("python_profile_specs", []): if not isinstance(raw_spec, str): raise CiError("projected python_profile_specs must contain strings") @@ -539,7 +557,7 @@ def _projected_profile_packages(self, projection: Path) -> tuple[str, ...] | Non elif value not in {"1", "true", "yes", "on"}: raise CiError(f"invalid projected profile prebuild flag: {parts[4]!r}") if prebuild: - family_requirements.append(parts[1]) + family_profiles.append((parts[0], parts[1])) registry_path = package_root / "python_profiles.toml" registry = tomllib.loads(registry_path.read_text(encoding="utf-8")) @@ -559,17 +577,15 @@ def _projected_profile_packages(self, projection: Path) -> tuple[str, ...] | Non f"projected Python profile {profile!r} has no requirements" ) generic_requirements.append((str(profile), value.strip())) - if not family_requirements and not any( - profile in selected_profiles for profile, _requirements in generic_requirements - ): - return None - requirements = [ - *family_requirements, - *(value for _profile, value in generic_requirements), + selected_generic = [ + item for item in generic_requirements if item[0] in selected_profiles ] + profiles = [*family_profiles, *selected_generic] + if not profiles: + return None result: set[str] = set() - for value in requirements: + for _profile, value in profiles: relative = Path(value) if relative.is_absolute() or ".." in relative.parts: raise CiError(f"projected Python profile has an unsafe requirements path: {value!r}") @@ -591,7 +607,10 @@ def _projected_profile_packages(self, projection: Path) -> tuple[str, ...] | Non result.add(line) if len(result) > 256: raise CiError("projected Python profiles declare more than 256 packages") - return tuple(sorted(result)) + return _PythonProfilePlan( + names=tuple(sorted(profile for profile, _requirements in profiles)), + packages=tuple(sorted(result)), + ) @staticmethod def _projected_selected_profile_names( @@ -630,6 +649,15 @@ def add_profiles(value: object, label: str) -> None: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if not isinstance(manifest, dict): raise CiError(f"projected E2E manifest must be an object: {manifest_path}") + index_path = manifest_path.parent.parent / "MODEL.toml" + e2e_index = ( + tomllib.loads(index_path.read_text(encoding="utf-8")) + if index_path.is_file() + else {} + ) + e2e_defaults = e2e_index.get("e2e_defaults", {}) + if not isinstance(e2e_defaults, dict): + raise CiError(f"projected E2E defaults must be an object: {index_path}") testcases = manifest.get("testcases", []) if not isinstance(testcases, list): raise CiError(f"projected E2E testcases must be a list: {manifest_path}") @@ -640,11 +668,21 @@ def add_profiles(value: object, label: str) -> None: cases.append({**manifest, **testcase}) for case in cases: add_profiles(case.get("execution_profiles"), "execution_profiles") + task_strategy = case.get("task_strategy") + task_defaults = ( + e2e_defaults.get(task_strategy, {}) + if isinstance(task_strategy, str) + else {} + ) + if not isinstance(task_defaults, dict): + raise CiError( + f"projected E2E defaults for {task_strategy!r} must be an object" + ) for section_name, selector_field in ( ("runtime_strategy_defaults", "runtime_strategy"), ("reference_backend_defaults", "reference_backend"), ): - selector = case.get(selector_field) + selector = case.get(selector_field, task_defaults.get(selector_field)) if not isinstance(selector, str) or not selector.strip(): continue section = registry.get(section_name, {}) From 7c33e48a18ddfbad55d4b0e68ea590f0a9960053 Mon Sep 17 00:00:00 2001 From: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> Date: Fri, 28 Aug 2026 20:46:44 -0700 Subject: [PATCH 7/7] fix(ci): name profile preparation user Profile containers run as the host UID, which may not exist in the image passwd database. Provide deterministic USER and LOGNAME values so package verification does not fall back to NSS lookup. Signed-off-by: yifeif-nv <277870278+yifeif-nv@users.noreply.github.com> --- tests/tools/test_model_proof_runner.py | 6 ++++++ tools/ci/model_proof.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/tests/tools/test_model_proof_runner.py b/tests/tools/test_model_proof_runner.py index 5ca3a16ef8..90dc52e4c2 100644 --- a/tests/tools/test_model_proof_runner.py +++ b/tests/tools/test_model_proof_runner.py @@ -1873,6 +1873,9 @@ def prepare(command: list[object], _log: Path, **_kwargs) -> int: assert "demo-package==1.0.0" in download assert f"src={packages},dst={PROFILE_PACKAGES_ROOT}" in download assert f"src={projection}" not in download + assert "HOME=/tmp" in download + assert "USER=trtmc-ci" in download + assert "LOGNAME=trtmc-ci" in download assert "NVIDIA_VISIBLE_DEVICES=void" in download assert "CUDA_VISIBLE_DEVICES=" in download @@ -1885,6 +1888,9 @@ def prepare(command: list[object], _log: Path, **_kwargs) -> int: assert "PIP_CONFIG_FILE=/dev/null" in install assert f"PIP_FIND_LINKS={PROFILE_PACKAGES_ROOT}" in install assert "PIP_NO_INDEX=1" in install + assert "HOME=/tmp" in install + assert "USER=trtmc-ci" in install + assert "LOGNAME=trtmc-ci" in install assert f"src={projection},dst=/src,readonly" in install assert f"src={profiles},dst={PREPARED_PROFILE_ROOT}" in install assert f"src={packages},dst={PROFILE_PACKAGES_ROOT},readonly" in install diff --git a/tools/ci/model_proof.py b/tools/ci/model_proof.py index d4dc14c19b..75280c5b27 100644 --- a/tools/ci/model_proof.py +++ b/tools/ci/model_proof.py @@ -492,6 +492,10 @@ def _prepare_python_profiles( "-e", "HOME=/tmp", "-e", + "USER=trtmc-ci", + "-e", + "LOGNAME=trtmc-ci", + "-e", "NVIDIA_VISIBLE_DEVICES=void", "-e", "CUDA_VISIBLE_DEVICES=", @@ -743,6 +747,10 @@ def _download_python_profile_packages( "-e", "HOME=/tmp", "-e", + "USER=trtmc-ci", + "-e", + "LOGNAME=trtmc-ci", + "-e", "NVIDIA_VISIBLE_DEVICES=void", "-e", "CUDA_VISIBLE_DEVICES=",