diff --git a/.dockerignore b/.dockerignore index 48b4cd9f95..f3bfb79371 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,25 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# The repository Dockerfile copies only the declarative Python profile source -# and its image-build helper. Keep the daemon context stable and small even on -# long-lived self-hosted runners with large build/test artifacts. +# The repository Dockerfile installs the stable base toolchain and copies only +# its model-agnostic package downloader. Family profiles are prepared per proof. * -!python/ -!python/tensorrt_model_connect/ -!python/tensorrt_model_connect/__init__.py -!python/tensorrt_model_connect/python_profiles.py -!python/tensorrt_model_connect/python_profiles.toml -!python/tensorrt_model_connect/families/ -!python/tensorrt_model_connect/families/__init__.py -!python/tensorrt_model_connect/families/*/ -!python/tensorrt_model_connect/families/*/MODEL.toml -!python/tensorrt_model_connect/families/*/python_profile_requirements/ -!python/tensorrt_model_connect/families/*/python_profile_requirements/*.lock.txt -!python/tensorrt_model_connect/families/*/python_profile_verify.py -!.github/ -!.github/scripts/ -!.github/scripts/build-python-profiles.py - -**/__pycache__/ -**/*.py[cod] +!tools/ +!tools/ci/ +!tools/ci/profile_downloader.py diff --git a/.github/scripts/build-python-profiles.py b/.github/scripts/build-python-profiles.py index 0e1d2b607a..0623ee4595 100644 --- a/.github/scripts/build-python-profiles.py +++ b/.github/scripts/build-python-profiles.py @@ -1,12 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Materialize every declared Python profile during the CI image build.""" +"""Prepare exact-pinned Python profiles before network-disabled execution.""" from __future__ import annotations +import argparse import importlib.util -import json import os import sys import types @@ -39,33 +39,34 @@ def _load_profile_api(): return module -def main() -> None: +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--profile", action="append", default=[]) + args = parser.parse_args(argv) profile_api = _load_profile_api() - names = profile_api.prebuilt_python_profile_names( + available = profile_api.prebuilt_python_profile_names( profile_api.load_python_profile_registry() ) - if not names: + if not available: raise SystemExit("no prebuilt Python profiles were declared") + requested = tuple(args.profile) + if requested: + if len(set(requested)) != len(requested): + raise SystemExit("requested Python profiles must be unique") + unknown = sorted(set(requested) - set(available)) + if unknown: + raise SystemExit("requested Python profiles are not prebuilt: " + ",".join(unknown)) + names = tuple(sorted(requested)) + else: + names = available base_python = os.environ.get("TRTMC_BASE_PYTHON", "/opt/venv/bin/python") - resolved: dict[str, dict[str, str]] = {} for name in names: python = profile_api.resolve_profile_python(name, base_python) ready = Path(python).parent.parent / ".ready" if not ready.is_file(): raise SystemExit(f"profile {name!r} was not marked ready: {ready}") - resolved[name] = {"python": python, "ready": str(ready)} - - manifest = { - "schema_version": 1, - "profiles": resolved, - } - root = profile_api.profile_root() - (root / ".image-ready.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - print("prebuilt_python_profiles=" + ",".join(names)) + print("prepared_python_profiles=" + ",".join(names)) if __name__ == "__main__": diff --git a/.github/scripts/write-model-proof-fallback-report.py b/.github/scripts/write-model-proof-fallback-report.py index a5e8d4206c..f0a063b6c9 100644 --- a/.github/scripts/write-model-proof-fallback-report.py +++ b/.github/scripts/write-model-proof-fallback-report.py @@ -9,6 +9,8 @@ import argparse import html import json +import os +import stat from pathlib import Path from typing import Sequence @@ -16,6 +18,8 @@ _DIAGNOSTIC_FILES = ( "host-error.log", "ci-image.log", + "python-profiles-prepare.log", + "python-profile-download.log", "console.log", "projection.stderr.log", "projection.json", @@ -23,6 +27,7 @@ "build.log", ) _MAX_DIAGNOSTIC_CHARS = 16_000 +_MAX_DIAGNOSTIC_BYTES = _MAX_DIAGNOSTIC_CHARS * 4 def _load_json(path: Path) -> dict[str, object]: @@ -37,14 +42,29 @@ def _diagnostics(root: Path) -> list[tuple[str, str]]: excerpts: list[tuple[str, str]] = [] for filename in _DIAGNOSTIC_FILES: path = root / filename - if not path.is_file(): + try: + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError: continue try: - text = path.read_text(encoding="utf-8", errors="replace") + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + continue + offset = max(0, metadata.st_size - _MAX_DIAGNOSTIC_BYTES) + os.lseek(descriptor, offset, os.SEEK_SET) + payload = os.read(descriptor, _MAX_DIAGNOSTIC_BYTES) except OSError: continue + finally: + os.close(descriptor) + text = payload.decode("utf-8", errors="replace")[-_MAX_DIAGNOSTIC_CHARS:] if text.strip(): - excerpts.append((filename, text[-_MAX_DIAGNOSTIC_CHARS:])) + excerpts.append((filename, text)) return excerpts diff --git a/Dockerfile b/Dockerfile index e5e32762ca..b477e08623 100644 --- a/Dockerfile +++ b/Dockerfile @@ -135,44 +135,20 @@ RUN pip install --force-reinstall \ # reference inference on the system cuBLAS instead of pip-installed CUDA libs. ENV LD_PRELOAD=/usr/local/cuda/lib64/libcublas.so.13 +# This model-agnostic downloader is part of the reviewed base runtime. It may +# fetch exact public PyPI artifacts, but never imports or builds package code. +COPY tools/ci/profile_downloader.py /opt/trtmc-profile-downloader.py + # Coverage tooling verification (run inside container): # python3 -m coverage --version && pytest --version && \ # python3 -m pytest --help | grep -- '--cov' && \ # gcovr --version && lcov --version && genhtml --version -# Build every declarative Python execution profile while network access is -# available. Family-owned declarations, lock files, and verification scripts -# are the package source of truth; python_profiles.py rejects non-exact pins and -# verifies every installed distribution before marking a profile ready. -FROM ci-common-base AS python-profile-builder - -ENV TRTMC_PYTHON_PROFILE_ROOT=/opt/trtmc-python-profiles -# sphn publishes no aarch64 wheel. Keep its Rust build toolchain in this -# throwaway builder stage; the final development stage receives only the -# verified profile. -RUN apt-get update && \ - apt-get install -y --no-install-recommends cargo rustc && \ - rm -rf /var/lib/apt/lists/* && \ - pip install "maturin==1.14.1" -# Avoid compiling profile-local CUDA extensions for every architecture known -# to a GPU-less Docker build. Keep 10.0 as the GB300 target. -COPY python/tensorrt_model_connect /opt/trtmc-profile-source/tensorrt_model_connect -COPY .github/scripts/build-python-profiles.py /opt/trtmc-build-python-profiles.py -RUN python3 /opt/trtmc-build-python-profiles.py \ - && chmod -R a+rX /opt/trtmc-python-profiles - -# Do not retain the full builder source tree in the development image. Only the -# verified virtual environments cross the stage boundary, so sibling model -# implementations cannot satisfy imports in an isolated source projection. +# Keep the reusable runtime independent of family-owned Python environments. +# Online CI preparation materializes the selected family's exact-pinned +# profiles before a network-disabled proof and mounts them read-only there. FROM ci-common-base AS ci-common -COPY --from=python-profile-builder \ - /opt/trtmc-python-profiles /opt/trtmc-python-profiles -ENV TRTMC_PYTHON_PROFILE_ROOT=/opt/trtmc-python-profiles -# Execution-profile environments are part of the dev-image contract. Rebuild -# the image after changing their lock or verification files. -ENV TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1 - # Keep the reusable common layer independent of every TensorRT release. The # version overlay below is the only stage allowed to add bindings, headers, or # native runtime libraries. diff --git a/python/tensorrt_model_connect/families/nemotron_h/MODEL.toml b/python/tensorrt_model_connect/families/nemotron_h/MODEL.toml index 544594edde..b32d4be6a4 100644 --- a/python/tensorrt_model_connect/families/nemotron_h/MODEL.toml +++ b/python/tensorrt_model_connect/families/nemotron_h/MODEL.toml @@ -7,6 +7,10 @@ module = "plugin" python_profile_specs = [ "nemotron_h_reference|families/nemotron_h/python_profile_requirements/nemotron_h_reference.lock.txt|families/nemotron_h/python_profile_verify.py|true", ] +python_profile_build_environment = [ + "nemotron_h_reference|CAUSAL_CONV1D_FORCE_BUILD|TRUE", + "nemotron_h_reference|MAMBA_FORCE_BUILD|TRUE", +] default_execution_profiles = [ "reference|nemotron_h_reference", ] diff --git a/python/tensorrt_model_connect/python_profiles.py b/python/tensorrt_model_connect/python_profiles.py index d2747c3df6..6151bcaa1e 100644 --- a/python/tensorrt_model_connect/python_profiles.py +++ b/python/tensorrt_model_connect/python_profiles.py @@ -45,6 +45,25 @@ re.IGNORECASE, ) _PROFILE_NAME_RE = re.compile(r"[a-z][a-z0-9_]*") +_BUILD_ENVIRONMENT_NAME_RE = re.compile(r"[A-Z][A-Z0-9_]*") +_FORBIDDEN_BUILD_ENVIRONMENT_NAMES = { + "HOME", + "LD_LIBRARY_PATH", + "LD_PRELOAD", + "PATH", + "PYTHONHOME", + "PYTHONPATH", +} +_FORBIDDEN_BUILD_ENVIRONMENT_PREFIXES = ( + "AWS_", + "AZURE_", + "GIT_", + "GOOGLE_", + "NVIDIA_", + "PIP_", + "SSH_", + "TRTMC_", +) _REGISTRY_KEYS = { "version", "profiles", @@ -55,6 +74,7 @@ _VENV_PROFILE_KEYS = { "kind", "prebuild", + "build_environment", "requirements", "system_site_packages", "verification_script", @@ -153,6 +173,7 @@ def family_python_profile_specs() -> dict[str, dict[str, object]]: with manifest.open("rb") as stream: raw = tomllib.load(stream) family_id = raw.get("id") or raw.get("plugin") or manifest.parent.name + family_profile_names: set[str] = set() raw_specs = raw.get("python_profile_specs", []) if not isinstance(raw_specs, list): raise ValueError( @@ -192,6 +213,49 @@ def family_python_profile_specs() -> dict[str, dict[str, object]]: "system_site_packages": system_site_packages, "prebuild": prebuild, } + family_profile_names.add(name) + raw_build_environment = raw.get("python_profile_build_environment", []) + if not isinstance(raw_build_environment, list): + raise ValueError( + f"python_profile_build_environment for family {family_id} must be a list" + ) + for entry in raw_build_environment: + if not isinstance(entry, str): + raise ValueError( + f"python_profile_build_environment for family {family_id} " + "must contain strings" + ) + parts = [part.strip() for part in entry.split("|", 2)] + if len(parts) != 3 or any(not part for part in parts): + raise ValueError( + f"Invalid python_profile_build_environment entry {entry!r} " + f"for family {family_id}; expected 'profile|NAME|value'" + ) + profile, name, value = parts + if profile not in family_profile_names: + raise ValueError( + f"python_profile_build_environment selects undeclared profile " + f"{profile!r} for family {family_id}" + ) + if ( + _BUILD_ENVIRONMENT_NAME_RE.fullmatch(name) is None + or name in _FORBIDDEN_BUILD_ENVIRONMENT_NAMES + or name.startswith(_FORBIDDEN_BUILD_ENVIRONMENT_PREFIXES) + ): + raise ValueError( + f"Python profile {profile!r} has unsafe build environment name {name!r}" + ) + if len(value) > 1024 or "\x00" in value or "\n" in value or "\r" in value: + raise ValueError( + f"Python profile {profile!r} has an unsafe build environment value" + ) + build_environment = dict(profiles[profile].get("build_environment", {})) + if name in build_environment: + raise ValueError( + f"Python profile {profile!r} declares build environment {name!r} twice" + ) + build_environment[name] = value + profiles[profile]["build_environment"] = build_environment return profiles @@ -260,6 +324,23 @@ def _validate_python_profile_registry(registry: Mapping[str, Any]) -> None: raise ValueError( f"Execution profile {name!r} field {field} must be a bool" ) + build_environment = raw_spec.get("build_environment", {}) + if not isinstance(build_environment, Mapping) or any( + not isinstance(name, str) + or _BUILD_ENVIRONMENT_NAME_RE.fullmatch(name) is None + or name in _FORBIDDEN_BUILD_ENVIRONMENT_NAMES + or name.startswith(_FORBIDDEN_BUILD_ENVIRONMENT_PREFIXES) + or not isinstance(value, str) + or not value + or len(value) > 1024 + or "\x00" in value + or "\n" in value + or "\r" in value + for name, value in build_environment.items() + ): + raise ValueError( + f"Execution profile {name!r} build_environment must contain safe strings" + ) requirements = raw_spec.get("requirements") if type(requirements) is not str: raise ValueError( @@ -334,7 +415,7 @@ def _validate_python_profile_registry(registry: Mapping[str, Any]) -> None: def prebuilt_python_profile_names( registry: Mapping[str, Any] | None = None, ) -> tuple[str, ...]: - """Return non-default profiles that belong in the shared CI image.""" + """Return non-default profiles prepared before network-disabled execution.""" selected = ( registry if registry is not None else load_python_profile_registry() ) @@ -669,6 +750,10 @@ def _materialize_venv_profile( ) verification_script = _read_package_text(verification_script_file).strip() system_site_packages = bool(spec.get("system_site_packages", True)) + build_environment = { + str(name): str(value) + for name, value in dict(spec.get("build_environment", {})).items() + } hash_input = "\n".join( [ @@ -678,6 +763,7 @@ def _materialize_venv_profile( requirements_text, verification_script, f"system_site_packages={int(system_site_packages)}", + json.dumps(build_environment, separators=(",", ":"), sort_keys=True), ] ).encode("utf-8") profile_hash = hashlib.sha256(hash_input).hexdigest()[:12] @@ -688,15 +774,14 @@ def _materialize_venv_profile( ready_path = env_dir / ".ready" lock_path = root / f"{profile_name}-{profile_hash}.lock" - # Model-proof containers mount the source read-only and disable networking. - # A matching image-baked profile therefore needs no writable lock or cache. + # Network-disabled proofs mount a separately prepared profile root read-only. if ready_path.is_file() and python_path.is_file(): return str(python_path.absolute()) if _prebuilt_only(): raise RuntimeError( f"Execution profile {profile_name!r} is not prebuilt for this source " - f"at {env_dir}. The CI image is stale or incomplete; rebuild it from " - "the current Dockerfile and declarative profile locks." + f"at {env_dir}. Prepare the declared profiles before entering the " + "network-disabled execution lane." ) root.mkdir(parents=True, exist_ok=True) @@ -726,6 +811,8 @@ def _materialize_venv_profile( _write_base_site_packages_overlay(base_python, str(tmp_python)) if requirements_text.strip(): + install_environment = _profile_install_environment() + install_environment.update(build_environment) _run_profile_command( [ str(tmp_python), @@ -741,7 +828,7 @@ def _materialize_venv_profile( ], description=f"install Python profile {profile_name!r}", timeout=_PROFILE_INSTALL_TIMEOUT_SECONDS, - env=_profile_install_environment(), + env=install_environment, ) _verify_exact_requirements( diff --git a/tests/tools/test_e2e_python_profiles.py b/tests/tools/test_e2e_python_profiles.py index 1105998d14..6726806c48 100644 --- a/tests/tools/test_e2e_python_profiles.py +++ b/tests/tools/test_e2e_python_profiles.py @@ -151,6 +151,9 @@ def test_resolve_profile_python_materializes_declared_venv(monkeypatch, tmp_path == python ) assert created == ["custom"] + monkeypatch.setenv(shared_profiles.PREBUILT_ONLY_ENV, "1") + assert shared_profiles.resolve_profile_python("custom", sys.executable) == python + assert created == ["custom"] def test_profile_source_builds_use_a_safe_default_job_limit(monkeypatch, tmp_path): @@ -183,12 +186,14 @@ def run_command(cmd, *, description, timeout=1800, **kwargs): "requirements": str(requirements), "system_site_packages": False, "verification_script": "print('verified')", + "build_environment": {"DEMO_FORCE_BUILD": "TRUE"}, }, sys.executable, ) install = next(call for call in commands if call[1].startswith("install ")) assert install[3]["env"]["MAX_JOBS"] == "4" + assert install[3]["env"]["DEMO_FORCE_BUILD"] == "TRUE" assert "PYTHONPATH" not in install[3]["env"] assert install[2] == 7200 verify = next(call for call in commands if call[1].startswith("verify ")) @@ -269,6 +274,28 @@ def test_family_profile_registry_is_fully_exact_pinned(): ) pins = shared_profiles._exact_pinned_requirements(requirements) assert pins, name + assert profiles["nemotron_h_reference"]["build_environment"] == { + "CAUSAL_CONV1D_FORCE_BUILD": "TRUE", + "MAMBA_FORCE_BUILD": "TRUE", + } + + +def test_profile_build_environment_rejects_package_source_overrides() -> None: + with pytest.raises(ValueError, match="safe strings"): + shared_profiles._validate_python_profile_registry( + { + "version": 1, + "profiles": { + "base": {"kind": "passthrough"}, + "unsafe": { + "kind": "venv", + "requirements": "python_profile_requirements/reference_common.lock.txt", + "verification_script": "pass", + "build_environment": {"PIP_INDEX_URL": "https://example.invalid"}, + }, + } + } + ) def test_profile_contract_has_one_family_owned_source_of_truth() -> None: @@ -287,13 +314,14 @@ def test_profile_contract_has_one_family_owned_source_of_truth() -> None: assert merged_names == shared_names | family_names -def test_lazy_profiles_are_excluded_from_the_shared_ci_image() -> None: +def test_lazy_profiles_are_excluded_from_offline_preparation() -> None: registry = shared_profiles.load_python_profile_registry() prebuilt = shared_profiles.prebuilt_python_profile_names(registry) assert "personaplex_full_duplex_evaluator" not in prebuilt assert "reference_common" in prebuilt + def test_profile_lock_rejects_non_exact_or_duplicate_requirements(): with pytest.raises(ValueError, match="exact name==version pins"): shared_profiles._exact_pinned_requirements("transformers>=4.48\n") @@ -450,7 +478,7 @@ def test_prebuilt_only_profile_fails_before_creating_a_runtime_cache( }, ) - with pytest.raises(RuntimeError, match="CI image is stale or incomplete"): + with pytest.raises(RuntimeError, match="Prepare the declared profiles"): shared_profiles.resolve_profile_python("custom", sys.executable) assert not profile_root.exists() diff --git a/tests/tools/test_ensure_ci_docker_image.py b/tests/tools/test_ensure_ci_docker_image.py index 89b1ff46c9..68d9762bd8 100644 --- a/tests/tools/test_ensure_ci_docker_image.py +++ b/tests/tools/test_ensure_ci_docker_image.py @@ -266,6 +266,7 @@ def _write_profile_fingerprint_repo(tmp_path: Path) -> tuple[Path, Path, Path]: shutil.copytree(REPO_ROOT / ".github" / "scripts", repo_root / ".github" / "scripts") shutil.copytree(REPO_ROOT / "tools" / "ci", repo_root / "tools" / "ci") shutil.copy2(REPO_ROOT / "tools" / "__init__.py", repo_root / "tools" / "__init__.py") + shutil.copy2(REPO_ROOT / ".dockerignore", repo_root / ".dockerignore") package_root = repo_root / "python" / "tensorrt_model_connect" families_root = package_root / "families" @@ -464,7 +465,7 @@ def test_matching_image_is_fully_validated_once_per_workflow_run(tmp_path: Path) assert "reused from this workflow run's verified image" in result.stdout -def test_missing_prebuilt_profiles_rebuilds_the_image(tmp_path: Path) -> None: +def test_base_image_ignores_family_profile_inventory(tmp_path: Path) -> None: bootstrap_result, bootstrap_env, bootstrap_log = _run_ensure_script( tmp_path / "bootstrap", existing_images={}, @@ -483,8 +484,8 @@ def test_missing_prebuilt_profiles_rebuilds_the_image(tmp_path: Path) -> None: ) assert result.returncode == 0, result.stderr - assert f"-t {resolved_image}" in docker_log - assert "prebuilt Python profiles differ" in result.stdout + assert f"-t {resolved_image}" not in docker_log + assert f"CI Docker image '{resolved_image}' already matches" in result.stdout def test_tensorrt_overlay_contract_reports_each_mismatch(tmp_path: Path) -> None: @@ -537,13 +538,13 @@ def test_source_contract_describes_parameterized_tensorrt_overlay(tmp_path: Path tensorrt_apt_version="11.2.1.2-1+cuda13.3", ) assert selected_contract["schema_version"] == 1 - assert selected_contract["environment_contract_version"] == 2 + assert selected_contract["environment_contract_version"] == 3 assert ( selected_contract["common_input_fingerprint"] == default_contract["common_input_fingerprint"] ) assert selected_contract["input_fingerprint"] != default_contract["input_fingerprint"] - assert selected_contract["python_profiles"] == ["demo", "reference_common"] + assert selected_contract["python_profiles"] == [] assert selected_contract["tensorrt"] == { "version": "11.2.1.2", "apt_version": "11.2.1.2-1+cuda13.3", @@ -563,7 +564,7 @@ def test_source_contract_describes_parameterized_tensorrt_overlay(tmp_path: Path } -def test_source_contract_loads_profiles_without_ambient_pythonpath(tmp_path: Path) -> None: +def test_source_contract_ignores_profiles_without_ambient_pythonpath(tmp_path: Path) -> None: repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) environment = os.environ.copy() environment.pop("PYTHONPATH", None) @@ -585,7 +586,7 @@ def test_source_contract_loads_profiles_without_ambient_pythonpath(tmp_path: Pat ) assert result.returncode == 0, result.stderr - assert result.stdout.strip() == "demo,reference_common" + assert result.stdout.strip() == "" def test_image_contract_cli_emits_canonical_contract_json(tmp_path: Path) -> None: @@ -611,7 +612,8 @@ def test_image_contract_cli_emits_canonical_contract_json(tmp_path: Path) -> Non assert result.returncode == 0, result.stderr contract = json.loads(result.stdout) - assert contract["environment_contract_version"] == 2 + assert contract["environment_contract_version"] == 3 + assert contract["python_profiles"] == [] assert contract["tensorrt"]["version"] == "11.2.1.2" assert contract["tensorrt"]["apt_version"] == "11.2.1.2-1+cuda13.3" @@ -636,7 +638,6 @@ def test_validate_image_contract_returns_the_verified_source_contract( "MODELOPT_VERSION": expected.modelopt, "NLOHMANN_JSON_HEADER": "present", "NEMO_PROMPT_RNNT": "available", - "PYTHON_PROFILES": expected.python_profiles, } monkeypatch.setattr(manager, "_query_fingerprint", lambda _: expected.fingerprint) monkeypatch.setattr(manager, "_query_versions", lambda _: actual) @@ -696,48 +697,20 @@ def test_source_contract_rejects_mixed_tensorrt_overlay_versions(tmp_path: Path) ) -def test_source_contract_rechecks_profile_asset_containment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) - outside = tmp_path / "outside.lock.txt" - outside.write_text("demo==1.0\n", encoding="utf-8") - manager = DockerImageManager(repo_root) - monkeypatch.setattr( - manager, - "_load_profile_registry", - lambda: ( - { - "version": 1, - "profiles": { - "demo": { - "kind": "venv", - "prebuild": True, - "requirements": str(outside), - } - }, - }, - ("demo",), - ), - ) - - with pytest.raises(CiError, match="unsafe requirements path"): - manager.source_contract() - - -def test_profile_sources_are_fingerprinted_and_repo_is_the_build_context() -> None: +def test_only_base_sources_are_fingerprinted_and_repo_is_the_build_context() -> None: script = SCRIPT.read_text(encoding="utf-8") assert "class DockerImageManager" in script assert "semantic_fingerprint" in script - assert 'b"python-profile-registry\\0"' in script - assert "assets: set[Path]" in script + assert 'b"ci-base-runtime\\0"' in script + assert "assets: set[Path]" not in script assert 'Path("tools/ci/process.py")' not in script - assert 'package_root / "python_profiles.py"' in script + assert 'Path("tools/ci/profile_downloader.py")' in script + assert 'package_root / "python_profiles.py"' not in script assert '"-f"' in script assert "str(self.config.dockerfile)" in script assert '"."' in script - assert "profile builder source leaked into the runtime image" in script + assert "profile builder source leaked into the runtime image" not in script assert '"--user"' in script assert '"65534:65534"' in script assert '"--read-only"' in script @@ -747,7 +720,7 @@ def test_profile_sources_are_fingerprinted_and_repo_is_the_build_context() -> No assert "source_contract_json" in script -def test_profile_fingerprint_ignores_manifest_comments_and_ownership_fields( +def test_base_fingerprint_ignores_manifest_comments_and_ownership_fields( tmp_path: Path, ) -> None: repo_root, manifest, _ = _write_profile_fingerprint_repo(tmp_path) @@ -774,7 +747,7 @@ def test_profile_fingerprint_ignores_manifest_comments_and_ownership_fields( assert ownership_changed == baseline -def test_profile_fingerprint_ignores_unrelated_family_loader_changes( +def test_base_fingerprint_ignores_unrelated_family_loader_changes( tmp_path: Path, ) -> None: repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) @@ -841,6 +814,48 @@ def test_profile_builder_does_not_execute_package_init(tmp_path: Path) -> None: assert "package init must not execute" not in result.stderr +def test_profile_builder_materializes_only_requested_profiles(tmp_path: Path) -> None: + package_root = tmp_path / "tensorrt_model_connect" + package_root.mkdir() + profile_pythons: dict[str, Path] = {} + for name in ("selected", "unselected"): + profile = tmp_path / name + (profile / "bin").mkdir(parents=True) + (profile / ".ready").touch() + profile_pythons[name] = profile / "bin/python" + serialized_pythons = {name: str(path) for name, path in profile_pythons.items()} + (package_root / "python_profiles.py").write_text( + "def load_python_profile_registry(): return {}\n" + "def prebuilt_python_profile_names(registry): " + "return ('selected', 'unselected')\n" + f"PROFILE_PYTHONS = {serialized_pythons!r}\n" + "def resolve_profile_python(name, base_python):\n" + " print('resolved_profile=' + name)\n" + " return str(PROFILE_PYTHONS[name])\n", + encoding="utf-8", + ) + env = os.environ.copy() + env["TRTMC_PYTHON_PROFILE_SOURCE"] = str(package_root) + + result = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / ".github/scripts/build-python-profiles.py"), + "--profile", + "selected", + ], + env=env, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "resolved_profile=selected" in result.stdout + assert "resolved_profile=unselected" not in result.stdout + assert "prepared_python_profiles=selected" in result.stdout + + def test_source_contract_does_not_execute_or_fingerprint_family_loader( tmp_path: Path, ) -> None: @@ -860,7 +875,7 @@ def test_source_contract_does_not_execute_or_fingerprint_family_loader( assert changed["input_fingerprint"] == baseline["input_fingerprint"] -def test_profile_fingerprint_ignores_registry_comments(tmp_path: Path) -> None: +def test_base_fingerprint_ignores_profile_registry_comments(tmp_path: Path) -> None: repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) baseline = _resolved_image_for_repo(tmp_path / "baseline", repo_root) @@ -874,7 +889,7 @@ def test_profile_fingerprint_ignores_registry_comments(tmp_path: Path) -> None: assert changed == baseline -def test_profile_fingerprint_ignores_lazy_profile_declarations(tmp_path: Path) -> None: +def test_base_fingerprint_ignores_lazy_profile_declarations(tmp_path: Path) -> None: repo_root, manifest, _ = _write_profile_fingerprint_repo(tmp_path) baseline = _resolved_image_for_repo(tmp_path / "baseline", repo_root) @@ -890,7 +905,7 @@ def test_profile_fingerprint_ignores_lazy_profile_declarations(tmp_path: Path) - assert changed == baseline -def test_profile_fingerprint_changes_for_semantic_profile_declaration( +def test_base_fingerprint_ignores_semantic_profile_declaration( tmp_path: Path, ) -> None: repo_root, manifest, _ = _write_profile_fingerprint_repo(tmp_path) @@ -906,10 +921,21 @@ def test_profile_fingerprint_changes_for_semantic_profile_declaration( ) changed = _resolved_image_for_repo(tmp_path / "profile-change", repo_root) - assert changed != baseline + assert changed == baseline + + manifest.write_text( + manifest.read_text(encoding="utf-8") + + 'python_profile_build_environment = ["demo|DEMO_FORCE_BUILD|TRUE"]\n', + encoding="utf-8", + ) + environment_changed = _resolved_image_for_repo( + tmp_path / "profile-environment-change", + repo_root, + ) + assert environment_changed == baseline -def test_profile_fingerprint_changes_for_referenced_profile_asset_content( +def test_base_fingerprint_ignores_referenced_profile_asset_content( tmp_path: Path, ) -> None: repo_root, _, requirements = _write_profile_fingerprint_repo(tmp_path) @@ -918,19 +944,19 @@ def test_profile_fingerprint_changes_for_referenced_profile_asset_content( requirements.write_text("demo-package==1.0.1\n", encoding="utf-8") changed = _resolved_image_for_repo(tmp_path / "asset-change", repo_root) - assert changed != baseline + assert changed == baseline @pytest.mark.parametrize( "relative_path", ( - Path("Dockerfile"), Path(".github/scripts/build-python-profiles.py"), Path("python/tensorrt_model_connect/python_profiles.py"), + Path("python/tensorrt_model_connect/python_profiles.toml"), Path("python/tensorrt_model_connect/families/demo/verify.py"), ), ) -def test_profile_fingerprint_changes_for_every_baked_recipe_input( +def test_base_fingerprint_ignores_profile_preparation_inputs( tmp_path: Path, relative_path: Path, ) -> None: @@ -944,6 +970,31 @@ def test_profile_fingerprint_changes_for_every_baked_recipe_input( changed = _resolved_image_for_repo(tmp_path / "recipe-change", repo_root) + assert changed == baseline + + +@pytest.mark.parametrize( + "relative_path", + ( + Path("Dockerfile"), + Path(".dockerignore"), + Path("tools/ci/profile_downloader.py"), + ), +) +def test_base_fingerprint_changes_for_base_recipe_inputs( + tmp_path: Path, + relative_path: Path, +) -> None: + repo_root, _, _ = _write_profile_fingerprint_repo(tmp_path) + baseline = _resolved_image_for_repo(tmp_path / "baseline", repo_root) + target = repo_root / relative_path + target.write_text( + target.read_text(encoding="utf-8") + "\n# Base environment change.\n", + encoding="utf-8", + ) + + changed = _resolved_image_for_repo(tmp_path / "base-change", repo_root) + assert changed != baseline diff --git a/tests/tools/test_github_actions_ci.py b/tests/tools/test_github_actions_ci.py index b64e64619f..b50f929685 100644 --- a/tests/tools/test_github_actions_ci.py +++ b/tests/tools/test_github_actions_ci.py @@ -712,11 +712,11 @@ def test_source_ci_image_uses_common_and_parameterized_tensorrt_overlay() -> Non assert "ghcr.io" not in dockerfile assert "TENSORRT_SDK_IMAGE" not in dockerfile assert "/opt/tensorrt/python" not in dockerfile + assert "COPY tools/ci/profile_downloader.py /opt/trtmc-profile-downloader.py" in dockerfile from_lines = [line for line in dockerfile.splitlines() if line.startswith("FROM ")] assert from_lines == [ "FROM ${CUDA_IMAGE} AS ci-common-base", - "FROM ci-common-base AS python-profile-builder", "FROM ci-common-base AS ci-common", "FROM ci-common AS ci-runtime", ] @@ -724,8 +724,9 @@ def test_source_ci_image_uses_common_and_parameterized_tensorrt_overlay() -> Non common = dockerfile.split("FROM ci-common-base AS ci-common", maxsplit=1)[1].split( "FROM ci-common AS ci-runtime", maxsplit=1 )[0] - assert "COPY --from=python-profile-builder" in common - assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1" in common + assert "COPY --from=python-profile-builder" not in common + assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY" not in dockerfile + assert "/opt/trtmc-python-profiles" not in dockerfile assert 'find_spec("tensorrt") is None' in common assert "NvInferVersion.h" in common assert "NvOnnxParser.h" in common diff --git a/tests/tools/test_model_ci.py b/tests/tools/test_model_ci.py index 2b482e0402..17d1d2d282 100644 --- a/tests/tools/test_model_ci.py +++ b/tests/tools/test_model_ci.py @@ -138,6 +138,11 @@ def _make_repo( _write(repo, "tests/runtime_strategy_matrix.yaml", "strategies: []\n") for source in sorted((REPO_ROOT / "tools/ci").glob("*.py")): _write(repo, f"tools/ci/{source.name}", f"# projected CI module: {source.name}\n") + _write( + repo, + ".github/scripts/build-python-profiles.py", + "#!/usr/bin/env python3\n", + ) _write( repo, ".github/scripts/write-model-proof-fallback-report.py", @@ -1394,6 +1399,7 @@ def test_projection_contains_only_selected_model_and_stable_git_blobs( assert fallback.is_file() assert not os.access(fallback, os.X_OK) for report_path in ( + ".github/scripts/build-python-profiles.py", "scripts/generate_e2e_report.py", "scripts/generate_e2e_report_assets/e2e_report.css", "scripts/generate_e2e_report_assets/e2e_report.js", @@ -1470,6 +1476,8 @@ def test_projection_includes_only_the_selected_family_adapter_subtrees( ) -> None: repo, _ = _make_repo(tmp_path) selected_paths = ( + "python/tensorrt_model_connect/families/model_b/python_profile_requirements/reference.lock.txt", + "python/tensorrt_model_connect/families/model_b/python_profile_verify.py", "python/tensorrt_model_connect/families/model_b/optimized_adapter/adapter.py", "python/tensorrt_model_connect/families/model_b/optimized_adapter/dependency.lock", "python/tensorrt_model_connect/families/model_b/optimized_adapter/profiles/example.toml", @@ -1481,6 +1489,18 @@ def test_projection_includes_only_the_selected_family_adapter_subtrees( _write(repo, path, "# selected model adapter\n") for path in sibling_paths: _write(repo, path, "# sibling model adapter\n") + for model in ("model_a", "model_b"): + manifest = repo / f"python/tensorrt_model_connect/families/{model}/MODEL.toml" + manifest.write_text( + manifest.read_text(encoding="utf-8") + + "python_profile_specs = [\n" + + ( + f' "{model}_reference|families/{model}/python_profile_requirements/' + f'reference.lock.txt|families/{model}/python_profile_verify.py|true",\n' + ) + + "]\n", + encoding="utf-8", + ) generic_host = "src/runtime/providers/optimized_runtime_host.cpp" _write(repo, generic_host, "// shared provider host\n") revision = _commit(repo, "add model-owned adapters") diff --git a/tests/tools/test_model_proof_runner.py b/tests/tools/test_model_proof_runner.py index beb26c7e1c..c31debacd7 100644 --- a/tests/tools/test_model_proof_runner.py +++ b/tests/tools/test_model_proof_runner.py @@ -7,6 +7,8 @@ from collections.abc import Callable, Iterator import fcntl +import hashlib +import io import json import os import re @@ -23,10 +25,17 @@ import pytest from tools.ci import gpu_lease as gpu_lease_module +from tools.ci import profile_downloader from tools.ci.context import CiContext from tools.ci.gpu_lease import GpuLease from tools.ci.model_reference_cache import ModelReferenceCacheWarmer -from tools.ci.model_proof import ModelProofRequest, ModelProofRunner, ModelReferenceCache +from tools.ci.model_proof import ( + PREPARED_PROFILE_ROOT, + PROFILE_PACKAGES_ROOT, + ModelProofRequest, + ModelProofRunner, + ModelReferenceCache, +) from tools.ci.model_proof_inner import ( ModelProofInnerPipeline, _classify_e2e_proof_kinds, @@ -111,6 +120,27 @@ def _write_successful_fake_docker(tmp_path: Path) -> tuple[Path, Path]: " exit 0\n" " ;;\n" " run)\n" + ' if [[ " $* " == *" /opt/trtmc-profile-downloader.py "* ]]; then\n' + " exit 0\n" + " fi\n" + ' if [[ " $* " == *" /src/.github/scripts/build-python-profiles.py "* ]]; then\n' + ' profile_root=""\n' + ' for argument in "$@"; do\n' + ' case "$argument" in\n' + ' type=bind,src=*,dst=/opt/trtmc-python-profiles)\n' + ' profile_root="${argument#type=bind,src=}"\n' + ' profile_root="${profile_root%,dst=/opt/trtmc-python-profiles}"\n' + " ;;\n" + " esac\n" + " done\n" + ' [ -n "$profile_root" ] || exit 95\n' + ' profile="$profile_root/reference_common-fake"\n' + ' mkdir -p "$profile/bin"\n' + ' ln -s /opt/venv/bin/python "$profile/bin/python"\n' + ' printf \'%s\\n\' \'profile=reference_common\' > "$profile/.ready"\n' + ' printf \'%s\\n\' \'{"schema_version":1,"profiles":{"reference_common":{"python":"/opt/trtmc-python-profiles/reference_common-fake/bin/python","ready":"/opt/trtmc-python-profiles/reference_common-fake/.ready"}}}\' > "$profile_root/.prepared-profiles.json"\n' + " exit 0\n" + " fi\n" ' if [[ " $* " == *" /src/scripts/warm_hf_cache.py "* ]]; then\n' ' mkdir -p "$FAKE_ARTIFACTS"\n' ' if [ "${FAKE_CACHE_EVIDENCE_MODE:-valid}" = escape ]; then\n' @@ -274,12 +304,17 @@ def _fake_proof_environment( return env -def _run_fake_proof(env: dict[str, str], output: Path) -> subprocess.CompletedProcess[str]: +def _run_fake_proof( + env: dict[str, str], + output: Path, + *, + model: str = "convbert", +) -> subprocess.CompletedProcess[str]: return subprocess.run( [ *RUNNER_COMMAND, "--model", - "convbert", + model, "--revision", "HEAD", "--output-dir", @@ -1775,6 +1810,416 @@ def test_model_proof_report_assets_are_inside_the_positive_projection() -> None: assert path.is_file(), path +def test_profile_preparation_uses_a_minimal_online_boundary( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + projection = tmp_path / "projection" + family = projection / "python/tensorrt_model_connect/families/demo" + family.mkdir(parents=True) + (family / "MODEL.toml").write_text( + 'id = "demo"\n' + 'python_profile_specs = [' + '"demo|families/demo/requirements.lock.txt|families/demo/verify.py|true"' + "]\n", + encoding="utf-8", + ) + (family / "requirements.lock.txt").write_text( + "demo-package==1.0.0\n", + encoding="utf-8", + ) + (family / "verify.py").write_text("import demo_package\n", encoding="utf-8") + package_root = projection / "python/tensorrt_model_connect" + (package_root / "python_profiles.toml").write_text( + 'version = 1\n[profiles.base]\nkind = "passthrough"\n', + encoding="utf-8", + ) + profiles = tmp_path / "python-profiles" + profiles.mkdir() + packages = tmp_path / "python-profile-packages" + packages.mkdir() + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + runner = ModelProofRunner( + CiContext(REPO_ROOT, {}), + ModelProofRequest(model="demo"), + ) + runner.artifacts_dir = artifacts + monkeypatch.setattr( + runner.context, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=0), + ) + commands: list[list[object]] = [] + + def prepare(command: list[object], _log: Path, **_kwargs) -> int: + commands.append(command) + return 0 + + monkeypatch.setattr(runner, "_run_logged", prepare) + + runner._prepare_python_profiles( + projection, + profiles, + packages, + "qualified-base@sha256:test", + ) + + assert len(commands) == 2 + download = " ".join(map(str, commands[0])) + install = " ".join(map(str, commands[1])) + assert "--network bridge" in download + assert "--runtime runc" in download + assert "/opt/trtmc-profile-downloader.py" in download + assert "demo-package==1.0.0" in download + assert f"src={packages},dst={PROFILE_PACKAGES_ROOT}" in download + assert f"src={projection}" not in download + assert "HOME=/tmp" in download + assert "USER=trtmc-ci" in download + assert "LOGNAME=trtmc-ci" in download + assert "NVIDIA_VISIBLE_DEVICES=void" in download + assert "CUDA_VISIBLE_DEVICES=" in download + + assert "--network none" in install + assert "--runtime runc" in install + assert "--read-only" in install + assert "--cap-drop ALL" in install + assert "--security-opt no-new-privileges" in install + assert "--ipc private" in install + assert "PIP_CONFIG_FILE=/dev/null" in install + assert f"PIP_FIND_LINKS={PROFILE_PACKAGES_ROOT}" in install + assert "PIP_NO_INDEX=1" in install + assert "HOME=/tmp" in install + assert "USER=trtmc-ci" in install + assert "LOGNAME=trtmc-ci" in install + assert f"src={projection},dst=/src,readonly" in install + assert f"src={profiles},dst={PREPARED_PROFILE_ROOT}" in install + assert f"src={packages},dst={PROFILE_PACKAGES_ROOT},readonly" in install + assert f"dst={PREPARED_PROFILE_ROOT},readonly" not in install + for command in (download, install): + assert "--gpus" not in command + assert "HF_TOKEN" not in command + assert "/var/run/docker.sock" not in command + assert "dst=/work" not in command + assert "dst=/artifacts" not in command + + +def test_profile_owning_family_runs_download_prepare_then_offline_proof( + tmp_path: Path, +) -> None: + fake_bin, docker_log = _write_successful_fake_docker(tmp_path) + output = tmp_path / "proof" + environment = _fake_proof_environment(tmp_path, fake_bin, docker_log, output) + + result = _run_fake_proof(environment, output, model="chronos_bolt") + + assert result.returncode == 0, result.stdout + result.stderr + runs = [ + line + for line in docker_log.read_text(encoding="utf-8").splitlines() + if line.startswith("run ") + ] + download_index = next( + index for index, command in enumerate(runs) if "/opt/trtmc-profile-downloader.py" in command + ) + prepare_index = next( + index + for index, command in enumerate(runs) + if "/src/.github/scripts/build-python-profiles.py" in command + ) + proof_index = next(index for index, command in enumerate(runs) if " --inner " in command) + assert download_index < prepare_index < proof_index + assert "--network bridge" in runs[download_index] + assert "--network none" in runs[prepare_index] + assert "--network none" in runs[proof_index] + assert f"dst={PREPARED_PROFILE_ROOT},readonly" in runs[proof_index] + assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1" in runs[proof_index] + + +@pytest.mark.parametrize("profile_owner", ("family", "generic")) +def test_empty_profile_lock_still_runs_offline_preparation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + profile_owner: str, +) -> None: + projection = tmp_path / "projection" + family = projection / "python/tensorrt_model_connect/families/demo" + family.mkdir(parents=True) + package_root = projection / "python/tensorrt_model_connect" + if profile_owner == "family": + (family / "MODEL.toml").write_text( + 'id = "demo"\n' + 'python_profile_specs = [' + '"demo|families/demo/requirements.lock.txt|families/demo/verify.py|true"' + "]\n", + encoding="utf-8", + ) + (family / "requirements.lock.txt").write_text( + "# no additional packages\n", + encoding="utf-8", + ) + (family / "verify.py").write_text("assert True\n", encoding="utf-8") + registry = 'version = 1\n[profiles.base]\nkind = "passthrough"\n' + else: + (family / "MODEL.toml").write_text( + 'id = "demo"\n' + 'default_execution_profiles = ["reference|generic"]\n', + encoding="utf-8", + ) + requirements = package_root / "python_profile_requirements" + requirements.mkdir() + (requirements / "generic.lock.txt").write_text( + "# no additional packages\n", + encoding="utf-8", + ) + (requirements / "unselected.lock.txt").write_text( + "unselected-package==1.0.0\n", + encoding="utf-8", + ) + registry = ( + 'version = 1\n[profiles.base]\nkind = "passthrough"\n' + '[profiles.generic]\nkind = "venv"\n' + 'requirements = "python_profile_requirements/generic.lock.txt"\n' + 'verification_script = "assert True"\n' + '[profiles.unselected]\nkind = "venv"\n' + 'requirements = "python_profile_requirements/unselected.lock.txt"\n' + 'verification_script = "assert True"\n' + ) + (package_root / "python_profiles.toml").write_text(registry, encoding="utf-8") + profiles = tmp_path / "python-profiles" + profiles.mkdir() + packages = tmp_path / "python-profile-packages" + packages.mkdir() + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + runner = ModelProofRunner( + CiContext(REPO_ROOT, {}), + ModelProofRequest(model="demo"), + ) + runner.artifacts_dir = artifacts + monkeypatch.setattr( + runner.context, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=0), + ) + commands: list[list[object]] = [] + monkeypatch.setattr( + runner, + "_run_logged", + lambda command, _log, **_kwargs: commands.append(command) or 0, + ) + + runner._prepare_python_profiles( + projection, + profiles, + packages, + "qualified-base@sha256:test", + ) + + assert len(commands) == 1 + preparation = " ".join(map(str, commands[0])) + assert "--network none" in preparation + assert "/src/.github/scripts/build-python-profiles.py" in preparation + assert "/opt/trtmc-profile-downloader.py" not in preparation + assert f"--profile {'demo' if profile_owner == 'family' else 'generic'}" in preparation + + +def test_e2e_defaults_select_a_generic_profile_for_preparation(tmp_path: Path) -> None: + projection = tmp_path / "projection" + package_root = projection / "python/tensorrt_model_connect" + requirements = package_root / "python_profile_requirements" + requirements.mkdir(parents=True) + (requirements / "generic.lock.txt").write_text( + "generic-package==1.2.3\n", + encoding="utf-8", + ) + (package_root / "python_profiles.toml").write_text( + 'version = 1\n[profiles.base]\nkind = "passthrough"\n' + '[profiles.generic]\nkind = "venv"\n' + 'requirements = "python_profile_requirements/generic.lock.txt"\n' + 'verification_script = "assert True"\n' + '[reference_backend_defaults.hf_transformers]\nreference = "generic"\n', + encoding="utf-8", + ) + e2e = projection / "tests/e2e/models/demo" + (e2e / "manifests").mkdir(parents=True) + (e2e / "MODEL.toml").write_text( + '[e2e_defaults.demo_task]\nreference_backend = "hf_transformers"\n', + encoding="utf-8", + ) + (e2e / "manifests/demo.json").write_text( + json.dumps({ + "name": "demo", + "task_strategy": "demo_task", + "testcases": [{"name": "demo"}], + }), + encoding="utf-8", + ) + runner = ModelProofRunner(CiContext(REPO_ROOT, {}), ModelProofRequest("demo")) + + plan = runner._projected_python_profile_plan(projection) + + assert plan is not None + assert plan.names == ("generic",) + assert plan.packages == ("generic-package==1.2.3",) + + +def test_offline_proof_consumes_prepared_profiles_read_only() -> None: + source = RUNNER.read_text(encoding="utf-8") + proof = source.split("def _run_proof_container(", maxsplit=1)[1].split( + "def _proof_environment", maxsplit=1 + )[0] + environment = source.split("def _proof_environment(", maxsplit=1)[1].split( + "def _reclaim_orphans", maxsplit=1 + )[0] + + assert "dst={PREPARED_PROFILE_ROOT},\"" in proof + assert '"readonly"' in proof + assert '"--network"' in proof and '"none"' in proof + assert '"TRTMC_PYTHON_PROFILE_PREBUILT_ONLY": "1"' in environment + assert '"TRTMC_PYTHON_PROFILE_ROOT": PREPARED_PROFILE_ROOT' in environment + assert '"PIP_NO_INDEX": "1"' in environment + assert source.index("self._prepare_python_profiles(") < source.index("self.lease = GpuLease(") + + +def test_profile_download_program_fetches_a_digest_verified_sdist( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + artifact = b"source archive" + digest = hashlib.sha256(artifact).hexdigest() + metadata = json.dumps( + { + "urls": [ + { + "packagetype": "sdist", + "filename": "demo-1.0.0.tar.gz", + "url": "https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz", + "digests": {"sha256": digest}, + } + ] + } + ).encode() + + class Response(io.BytesIO): + def __init__(self, payload: bytes, url: str): + super().__init__(payload) + self.url = url + + def geturl(self) -> str: + return self.url + + def urlopen(request, timeout): + del timeout + url = str(request.full_url) + return Response( + metadata if url.endswith("/demo/1.0.0/json") else artifact, + url, + ) + + monkeypatch.setattr(profile_downloader.urllib.request, "urlopen", urlopen) + monkeypatch.setattr( + profile_downloader.subprocess, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=1), + ) + + profile_downloader.main([str(tmp_path), "demo==1.0.0"]) + + assert (tmp_path / "demo-1.0.0.tar.gz").read_bytes() == artifact + + +@pytest.mark.parametrize( + ("artifact_url", "expected_digest", "message"), + ( + ( + "https://example.invalid/demo-1.0.0.tar.gz", + hashlib.sha256(b"source archive").hexdigest(), + "untrusted source URL", + ), + ( + "https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz", + "0" * 64, + "digest mismatch", + ), + ), +) +def test_profile_download_program_rejects_untrusted_sdist_records( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + artifact_url: str, + expected_digest: str, + message: str, +) -> None: + artifact = b"source archive" + metadata = json.dumps( + { + "urls": [ + { + "packagetype": "sdist", + "filename": "demo-1.0.0.tar.gz", + "url": artifact_url, + "digests": {"sha256": expected_digest}, + } + ] + } + ).encode() + + class Response(io.BytesIO): + def __init__(self, payload: bytes, url: str): + super().__init__(payload) + self.url = url + + def geturl(self) -> str: + return self.url + + def urlopen(request, timeout): + del timeout + url = str(request.full_url) + return Response(metadata if url.endswith("/demo/1.0.0/json") else artifact, url) + + monkeypatch.setattr(profile_downloader.urllib.request, "urlopen", urlopen) + monkeypatch.setattr( + profile_downloader.subprocess, + "run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=1), + ) + + with pytest.raises(RuntimeError, match=message): + profile_downloader.main([str(tmp_path), "demo==1.0.0"]) + + assert not (tmp_path / "demo-1.0.0.tar.gz").exists() + + +def test_projected_profile_packages_include_source_only_nemotron_dependencies( + tmp_path: Path, +) -> None: + projection = tmp_path / "projection" + package = projection / "python/tensorrt_model_connect" + family = package / "families/nemotron_h" + family.mkdir(parents=True) + shutil.copy2( + REPO_ROOT / "python/tensorrt_model_connect/families/nemotron_h/MODEL.toml", + family / "MODEL.toml", + ) + shutil.copytree( + REPO_ROOT + / "python/tensorrt_model_connect/families/nemotron_h/python_profile_requirements", + family / "python_profile_requirements", + ) + (package / "python_profiles.toml").write_text( + 'version = 1\n[profiles.base]\nkind = "passthrough"\n', + encoding="utf-8", + ) + runner = ModelProofRunner(CiContext(REPO_ROOT, {}), ModelProofRequest("nemotron_h")) + + plan = runner._projected_python_profile_plan(projection) + + assert plan is not None + assert "mamba-ssm==2.3.2.post1" in plan.packages + assert "causal-conv1d==1.6.2.post1" in plan.packages + + def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: artifacts = tmp_path / "artifacts" artifacts.mkdir() @@ -1782,6 +2227,16 @@ def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: "model-ci: error: unknown model \n", encoding="utf-8", ) + (artifacts / "python-profiles-prepare.log").write_text( + "profile install failed\n", + encoding="utf-8", + ) + (artifacts / "console.log").write_text( + "discarded-prefix\n" + ("x" * 20_000) + "\nbounded-tail\n", + encoding="utf-8", + ) + (artifacts / "configure.log").symlink_to("/dev/zero") + os.mkfifo(artifacts / "build.log") result = subprocess.run( [ @@ -1805,6 +2260,7 @@ def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: capture_output=True, text=True, check=False, + timeout=5, ) assert result.returncode == 0, result.stderr @@ -1812,6 +2268,10 @@ def test_fallback_writer_embeds_host_diagnostics(tmp_path: Path) -> None: status = json.loads((artifacts / "model-proof-status.json").read_text(encoding="utf-8")) assert "host-error.log" in report assert "unknown model <unsafe>" in report + assert "python-profiles-prepare.log" in report + assert "profile install failed" in report + assert "bounded-tail" in report + assert "discarded-prefix" not in report assert status["outcome"] == "failed" assert status["exit_code"] == 2 diff --git a/tests/tools/test_selected_wheel_runtime.py b/tests/tools/test_selected_wheel_runtime.py index ca6a5ba3ca..bc78ef97ec 100644 --- a/tests/tools/test_selected_wheel_runtime.py +++ b/tests/tools/test_selected_wheel_runtime.py @@ -211,6 +211,8 @@ def test_model_proof_mounts_selected_wheels_read_only_and_forwards_contract( runner.artifacts_dir = tmp_path / "artifacts" runner.artifacts_dir.mkdir() runner.revision = "a" * 40 + python_profiles = tmp_path / "python-profiles" + python_profiles.mkdir() captured: list[list[object]] = [] monkeypatch.setattr( context, @@ -230,9 +232,16 @@ def test_model_proof_mounts_selected_wheels_read_only_and_forwards_contract( "fixture-image", SimpleNamespace(reference_cache=None), None, + python_profiles, ) command = captured[0] + assert ( + f"type=bind,src={python_profiles},dst=/opt/trtmc-python-profiles,readonly" + in command + ) + assert "TRTMC_PYTHON_PROFILE_ROOT=/opt/trtmc-python-profiles" in command + assert "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1" in command assert f"type=bind,src={selected.resolve()},dst=/selected-wheel,readonly" in command assert "TRTMC_SELECTED_WHEEL_DIR=/selected-wheel" in command assert f"TRTMC_SELECTED_WHEEL_PYTHON_TAG={PYTHON_TAG}" in command diff --git a/tools/ci/README.md b/tools/ci/README.md index 1c78e8e6e0..7e2b36d245 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -158,12 +158,20 @@ The host half, `ModelProofRunner`, performs trusted setup: platform files, but no peer model source. 3. Select the model-owned runtime, Python tests, E2E cases, resource class, and optional reference checkout. -4. Prepare only the selected Hugging Face repositories and reflink them into a +4. In a restricted networked `runc` container, use the downloader embedded in + the reviewed base image to fetch only the selected family's exact-pinned + package artifacts. This step executes no contributor builder, verifier, or + package source code. +5. In a second `--network none`, no-GPU `runc` container, install those local + artifacts, execute the projected profile verifier, and publish the + proof-private virtual environments. +6. Prepare only the selected Hugging Face repositories and reflink them into a proof-private cache view. Premerge downloads a missing snapshot into the current host cache; nightly model proofs reuse the cache prepared by the separate Nightly cache-warm job. -5. Acquire either shared GPU slots or a whole GPU through `GpuLease`. -6. Start a read-only, network-disabled proof container. +7. Acquire either shared GPU slots or a whole GPU through `GpuLease`. +8. Start a read-only, network-disabled proof container with the prepared + profile directory mounted read-only. The container half, `ModelProofInnerPipeline`, then runs linearly: @@ -334,18 +342,15 @@ the producing class remains the source of truth for optional evidence fields. ### `docker_image.py` -- **Functionality / units:** `DockerImageManager` fingerprints image inputs, +- **Functionality / units:** `DockerImageManager` fingerprints stable base-image inputs, serializes concurrent builds with `WorkflowImageLock`, verifies dependency - versions, exposes Runtime Contract v2, and reuses only a matching local image. + versions, exposes Runtime Contract v3, and reuses only a matching local image. - **Inputs:** The common fingerprint includes the Dockerfile, `.dockerignore`, - profile builder and registry implementations, the normalized declarations of - every prebuilt Python profile, and their referenced lock and verification - files. The overlay fingerprint adds the exact TensorRT Python and APT - versions. Family metadata remains family-owned; comments, ownership fields, - lazy profiles, the general family loader, and package `__init__.py` metadata - are deliberately excluded because they do not change the baked environment. - The profile builder loads its narrow API through a synthetic package, so - package initialization is also absent from the actual image-build path. + and the model-agnostic PyPI artifact downloader copied into the image. The + overlay fingerprint adds the exact TensorRT Python and APT versions. + Family profile declarations, locks, verifiers, and preparation code are + deliberately excluded because their environments are not baked into the + base image. Contract-producer and CLI control-plane files are reviewed separately and do not perturb the semantic runtime fingerprint when their output is unchanged. - **Outputs:** Returns an immutable Docker ID shaped as @@ -354,13 +359,28 @@ the producing class remains the source of truth for optional evidence fields. `image_ref=sha256:...` through `GITHUB_OUTPUT`, and maintains a local verification stamp. `python3 -m tools.ci image contract` prints the complete canonical contract JSON, including the full common and overlay fingerprints - and `environment_contract_version=2`; callers may select an exact overlay + and `environment_contract_version=3`; callers may select an exact overlay with `--tensorrt-version` and `--tensorrt-apt-version`. - **Boundary:** It proves image identity and contents. It neither starts a container nor chooses a CI stage. Local image verification is serialized by a six-hour default lock budget, overridable with `TRTMC_CI_IMAGE_LOCK_TIMEOUT`. +### `profile_downloader.py` + +- **Functionality / units:** Downloads compatible wheels for exact public PyPI + pins and falls back to the release's unique source archive when no wheel is + available, without importing or building downloaded package code. +- **Inputs:** A proof-private destination plus validated `name==version` pins. + Network access is fixed to public PyPI and its package CDN; redirects and + source-archive SHA-256 digests are checked before publication. +- **Outputs:** A bounded directory of wheel and source-distribution artifacts + consumed later through `PIP_NO_INDEX=1` and `PIP_FIND_LINKS`. +- **Boundary:** This is the only online Python-profile operation. The downloader + is copied into and fingerprinted with the reviewed base image; installation, + source builds, and contributor verifiers run only in the separate offline + preparation container. + ### `container.py` - **Functionality / units:** `ContainerConfig` resolves run identity, workspace, @@ -634,26 +654,33 @@ the producing class remains the source of truth for optional evidence fields. ### `model_proof.py` -- **Functionality / units:** `ModelProofRunner` performs trusted host setup; +- **Functionality / units:** `ModelProofRunner` performs trusted host setup, + prepares projected family Python profiles in a restricted networked container; `ModelReferenceCache` first ensures the selected pinned checkout is present, then copies only that model-owned reference checkout; `ModelProofContainerCleaner` removes containers matching exact run labels. - **Inputs:** `ModelProofRequest {model, suite, revision, output_dir}`, full repository checkout, CI image, shared HF/reference cache roots, workflow identity, and model-proof GPU settings. -- **Outputs:** A positive `projection/`, proof-private `work/`, and +- **Outputs:** A positive `projection/`, proof-private `python-profiles/` when + the family declares profiles, `work/`, and `artifacts/` containing at least `selection.json`, `gpu-lease.json`, cache/reference evidence, `console.log`, `proof.json`, and `model-proof-report.html`. Host failures still attempt a fallback report. - **Boundary:** This is the trusted host/security boundary. It may read shared - caches and Docker state, but model build and inference occur only in the - network-disabled inner container. + caches and Docker state. The reviewed online profile downloader receives no + source, GPU, secrets, or shared cache and only fetches exact public PyPI artifacts. + Package installation, source builds, and the projected verifier run in a + separate network-disabled `runc` container with read-only source. Model build + and inference occur only in the network-disabled inner container, which + consumes the profile directory read-only. ### `model_proof_inner.py` - **Functionality / units:** `ModelProofInnerPipeline` runs the linear proof; `ProofStatus` records every phase so report generation can fail closed. -- **Inputs:** Read-only projected source at `/src`, writable `/work`, output +- **Inputs:** Read-only projected source at `/src`, prepared Python profiles + mounted read-only at `/opt/trtmc-python-profiles`, writable `/work`, output mount `/artifacts`, selected offline HF cache, optional private reference tree, one visible GPU, lease environment fields, and `ModelProofRequest`. - **Outputs:** Build/test/reference evidence plus these certification records: diff --git a/tools/ci/docker_image.py b/tools/ci/docker_image.py index 08c59287d0..0906ca0ad2 100644 --- a/tools/ci/docker_image.py +++ b/tools/ci/docker_image.py @@ -10,13 +10,10 @@ import fcntl import hashlib -import importlib.util import json import os import re -import sys import time -import types from dataclasses import dataclass from pathlib import Path @@ -24,7 +21,7 @@ FINGERPRINT_LABEL = "org.nvidia.trtmc.ci-input-fingerprint" -ENVIRONMENT_CONTRACT_VERSION = 2 +ENVIRONMENT_CONTRACT_VERSION = 3 IMMUTABLE_IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}") EXACT_TENSORRT_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+") EXACT_APT_VERSION = re.compile(r"[0-9][0-9A-Za-z.+:~_-]*") @@ -87,7 +84,6 @@ class ImageRequirements: tensorrt: str tensorrt_apt: str modelopt: str - python_profiles: str @property def python_distributions(self) -> dict[str, str]: @@ -109,7 +105,9 @@ def contract(self) -> dict[str, object]: "common_input_fingerprint": self.common_fingerprint, "input_fingerprint": self.fingerprint, "modelopt_version": self.modelopt, - "python_profiles": self.python_profiles.split(","), + # Kept as an empty compatibility field for runtime-catalog readers. + # Family-owned profiles are prepared per proof, not baked here. + "python_profiles": [], "tensorrt": { "version": self.tensorrt, "apt_version": self.tensorrt_apt, @@ -207,9 +205,8 @@ def ensure(self) -> str: print( f"CI Docker image '{image}' verified: TensorRT {versions['TENSORRT_VERSION']}, " f"exact Python, APT header, C++ header, and native runtime contracts, modelopt " - f"{versions['MODELOPT_VERSION']}, nlohmann/json headers, NeMo prompt RNN-T and " - f"prebuilt Python profiles ({versions['PYTHON_PROFILES']}) present, " - f"image {image_id}" + f"{versions['MODELOPT_VERSION']}, nlohmann/json headers, and NeMo prompt RNN-T " + f"present, image {image_id}" ) return image_id @@ -263,29 +260,10 @@ def _read_requirements( tensorrt_version: str | None = None, tensorrt_apt_version: str | None = None, ) -> ImageRequirements: - registry, profile_names = self._load_profile_registry() - profiles = registry["profiles"] - expected_profiles = ",".join(profile_names) - if not expected_profiles: - raise CiError("No prebuilt Python execution profiles were declared") - - package_root = Path("python/tensorrt_model_connect") - assets: set[Path] = set() - prebuilt_profiles = {name: profiles[name] for name in profile_names} - for spec in prebuilt_profiles.values(): - if not isinstance(spec, dict): - continue - for field in ("requirements", "verification_script_file"): - value = str(spec.get(field, "") or "").strip() - if value: - assets.add(self._profile_asset_input(package_root, value, field)) - inputs = { self.config.dockerfile, Path(".dockerignore"), - Path(".github/scripts/build-python-profiles.py"), - package_root / "python_profiles.py", - *assets, + Path("tools/ci/profile_downloader.py"), } dockerfile_text = (self.config.repository / self.config.dockerfile).read_text( encoding="utf-8" @@ -306,8 +284,6 @@ def _read_requirements( ) common_semantic_contract = { "environment_contract_version": ENVIRONMENT_CONTRACT_VERSION, - "version": registry.get("version"), - "profiles": prebuilt_profiles, } common_semantic_payload = json.dumps( common_semantic_contract, @@ -340,68 +316,11 @@ def _read_requirements( tensorrt, tensorrt_apt, modelopt, - expected_profiles, ) - def _profile_asset_input( - self, - package_root: Path, - path_spec: str, - field: str, - ) -> Path: - relative = Path(path_spec) - if relative.is_absolute() or ".." in relative.parts: - raise CiError(f"Python profile has an unsafe {field} path: {path_spec!r}") - absolute_root = (self.config.repository / package_root).resolve() - resolved = (absolute_root / relative).resolve() - try: - resolved.relative_to(absolute_root) - except ValueError as error: - raise CiError( - f"Python profile has an unsafe {field} path: {path_spec!r}" - ) from error - if not resolved.is_file(): - raise CiError( - f"Python profile references a missing {field} asset: {path_spec!r}" - ) - return package_root / relative - - def _load_profile_registry(self) -> tuple[dict[str, object], tuple[str, ...]]: - package_name = "tensorrt_model_connect" - package_root = self.config.repository / "python" / package_name - module_name = f"{package_name}.python_profiles" - previous_modules = { - name: module - for name, module in sys.modules.items() - if name == package_name or name.startswith(f"{package_name}.") - } - for name in previous_modules: - sys.modules.pop(name, None) - package = types.ModuleType(package_name) - package.__package__ = package_name - package.__path__ = [str(package_root)] - sys.modules[package_name] = package - try: - spec = importlib.util.spec_from_file_location( - module_name, - package_root / "python_profiles.py", - ) - if spec is None or spec.loader is None: - raise CiError("Could not load the Source Python profile registry") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - registry = module.load_python_profile_registry() - return registry, module.prebuilt_python_profile_names(registry) - finally: - for name in tuple(sys.modules): - if name == package_name or name.startswith(f"{package_name}."): - sys.modules.pop(name, None) - sys.modules.update(previous_modules) - def _fingerprint_inputs(self, inputs: tuple[Path, ...], semantic: str) -> str: digest = hashlib.sha256() - digest.update(b"python-profile-registry\0") + digest.update(b"ci-base-runtime\0") digest.update(semantic.encode("ascii") + b"\n") for relative in inputs: digest.update(str(relative).encode("utf-8") + b"\0") @@ -595,21 +514,6 @@ def _query_versions(self, image: str) -> dict[str, str]: print("MODELOPT_VERSION=" + metadata.version("nvidia-modelopt")) print("NLOHMANN_JSON_HEADER=" + ("present" if Path("/usr/include/nlohmann/json.hpp").is_file() else "missing")) print("NEMO_PROMPT_RNNT=available") -manifest_path = Path("/opt/trtmc-python-profiles/.image-ready.json") -manifest = json.loads(manifest_path.read_text(encoding="utf-8")) -if Path("/opt/trtmc-profile-source").exists(): - raise SystemExit("profile builder source leaked into the runtime image") -profiles = manifest.get("profiles") -if not isinstance(profiles, dict) or not profiles: - raise SystemExit("prebuilt Python profile manifest is empty or invalid") -for name, record in profiles.items(): - if not isinstance(record, dict): - raise SystemExit(f"invalid prebuilt Python profile record: {name}") - python = Path(str(record.get("python", ""))) - ready = Path(str(record.get("ready", ""))) - if not python.is_file() or not ready.is_file(): - raise SystemExit(f"prebuilt Python profile is incomplete: {name}") -print("PYTHON_PROFILES=" + ",".join(sorted(profiles))) """ result = self.commands.run( [ @@ -678,12 +582,6 @@ def _version_mismatches(actual: dict[str, str], expected: ImageRequirements) -> reasons.append("nlohmann/json development headers are missing") if actual.get("NEMO_PROMPT_RNNT") != "available": reasons.append("required NeMo prompt RNN-T capability is missing") - if actual.get("PYTHON_PROFILES") != expected.python_profiles: - reasons.append( - "prebuilt Python profiles differ: image has " - f"'{actual.get('PYTHON_PROFILES', 'missing')}', source expects " - f"'{expected.python_profiles}'" - ) return reasons def _build(self, image: str, expected: ImageRequirements, reasons: list[str]) -> None: diff --git a/tools/ci/model_proof.py b/tools/ci/model_proof.py index 54c1bedbfc..75280c5b27 100644 --- a/tools/ci/model_proof.py +++ b/tools/ci/model_proof.py @@ -18,6 +18,11 @@ from dataclasses import dataclass from pathlib import Path +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python 3.10 compatibility. + import tomli as tomllib + from .context import CiContext from .gpu_lease import GpuLease from .model_reference_cache import ModelReferenceCacheWarmer, ModelReferenceContract @@ -56,6 +61,28 @@ raise """ +PREPARED_PROFILE_ROOT = "/opt/trtmc-python-profiles" +PROFILE_PACKAGES_ROOT = "/opt/trtmc-python-profile-packages" +_EXACT_PROFILE_REQUIREMENT_RE = re.compile( + r"^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[A-Za-z0-9,._-]+\])?==([^\s;]+)$" +) +_EXACT_PROFILE_VERSION_RE = re.compile( + r"(?:[0-9]+!)?[0-9]+(?:\.[0-9]+)*" + r"(?:(?:a|b|rc)[0-9]+)?" + r"(?:\.post[0-9]+)?" + r"(?:\.dev[0-9]+)?" + r"(?:\+[A-Za-z0-9]+(?:[._-][A-Za-z0-9]+)*)?", + re.IGNORECASE, +) + + +@dataclass(frozen=True) +class _PythonProfilePlan: + """Exact projected profiles and packages prepared before offline proof.""" + + names: tuple[str, ...] + packages: tuple[str, ...] + @dataclass(frozen=True) class ModelProofRequest: @@ -274,8 +301,10 @@ def _run_host(self) -> None: projection = output / "projection" self.artifacts_dir = output / "artifacts" work = output / "work" + python_profiles = output / "python-profiles" + python_profile_packages = output / "python-profile-packages" output.mkdir(parents=True, exist_ok=True) - for path in (self.artifacts_dir, work): + for path in (self.artifacts_dir, work, python_profiles, python_profile_packages): if path.exists(): shutil.rmtree(path) path.mkdir(parents=True) @@ -305,6 +334,12 @@ def _run_host(self) -> None: ["docker", "image", "inspect", image], check=False, capture_output=True ).returncode: raise CiError(f"CI image is not present: {image}") + self._prepare_python_profiles( + projection, + python_profiles, + python_profile_packages, + image, + ) runtime_model = str(selection.payload["owners"]["runtime"]) validation_container = self._base_container_name() + "-validation-data" self.container_name = validation_container @@ -351,6 +386,7 @@ def _run_host(self) -> None: image, selection, validation_dir, + python_profiles, ) for name in ("proof.json", "model-proof-report.html"): if not (self.artifacts_dir / name).is_file(): @@ -393,6 +429,352 @@ def _project(self, projection: Path) -> None: if not (projection / ".trtmc-model-projection.json").is_file(): raise CiError("model_ci.py did not produce a projection manifest") + def _prepare_python_profiles( + self, + projection: Path, + profile_dir: Path, + package_dir: Path, + image: str, + ) -> None: + """Materialize projected profiles online for one later offline proof.""" + assert self.artifacts_dir is not None + plan = self._projected_python_profile_plan(projection) + if plan is None: + return + if plan.packages: + self._download_python_profile_packages( + package_dir, + image, + plan.packages, + ) + name = self._base_container_name() + "-python-profiles" + self.container_name = name + self.context.run(["docker", "rm", "-f", name], check=False, capture_output=True) + command = [ + "timeout", + "--kill-after=2m", + "6h", + "docker", + "run", + "--rm", + "--name", + name, + *self._job_labels(), + "--read-only", + "--network", + "none", + "--runtime", + "runc", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--ipc", + "private", + "--pids-limit", + "512", + "--memory", + "48g", + "--cpus", + "8", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--mount", + f"type=bind,src={projection},dst=/src,readonly", + "--mount", + f"type=bind,src={profile_dir},dst={PREPARED_PROFILE_ROOT}", + "--mount", + f"type=bind,src={package_dir},dst={PROFILE_PACKAGES_ROOT},readonly", + "--tmpfs", + "/tmp:rw,exec,nosuid,nodev,size=8g", + "--workdir", + "/src", + "-e", + "HOME=/tmp", + "-e", + "USER=trtmc-ci", + "-e", + "LOGNAME=trtmc-ci", + "-e", + "NVIDIA_VISIBLE_DEVICES=void", + "-e", + "CUDA_VISIBLE_DEVICES=", + "-e", + "PYTHONDONTWRITEBYTECODE=1", + "-e", + "PIP_DISABLE_PIP_VERSION_CHECK=1", + "-e", + "PIP_CONFIG_FILE=/dev/null", + "-e", + f"PIP_FIND_LINKS={PROFILE_PACKAGES_ROOT}", + "-e", + "PIP_NO_CACHE_DIR=1", + "-e", + "PIP_NO_INDEX=1", + "-e", + "TRTMC_PYTHON_PROFILE_SOURCE=/src/python/tensorrt_model_connect", + "-e", + f"TRTMC_PYTHON_PROFILE_ROOT={PREPARED_PROFILE_ROOT}", + "-e", + "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=0", + image, + "/opt/venv/bin/python", + "/src/.github/scripts/build-python-profiles.py", + *(item for name in plan.names for item in ("--profile", name)), + ] + result = self._run_logged( + command, + self.artifacts_dir / "python-profiles-prepare.log", + max_output_chars=2_000_000, + ) + if result: + raise CiError( + f"Python profile preparation failed for {self.request.model} (exit {result})" + ) + + def _projected_python_profile_plan( + self, + projection: Path, + ) -> _PythonProfilePlan | None: + """Return selected profiles and pins, or None when none need preparation.""" + package_root = projection / "python/tensorrt_model_connect" + manifests = sorted((package_root / "families").glob("*/MODEL.toml")) + if len(manifests) > 1: + raise CiError("projected Python ownership contains multiple families") + family = ( + tomllib.loads(manifests[0].read_text(encoding="utf-8")) + if manifests + else {} + ) + family_profiles: list[tuple[str, str]] = [] + for raw_spec in family.get("python_profile_specs", []): + if not isinstance(raw_spec, str): + raise CiError("projected python_profile_specs must contain strings") + parts = [part.strip() for part in raw_spec.split("|")] + if len(parts) not in {3, 4, 5} or any(not part for part in parts[:3]): + raise CiError(f"invalid projected python_profile_specs entry: {raw_spec!r}") + prebuild = True + if len(parts) == 5: + value = parts[4].lower() + if value in {"0", "false", "no", "off"}: + prebuild = False + elif value not in {"1", "true", "yes", "on"}: + raise CiError(f"invalid projected profile prebuild flag: {parts[4]!r}") + if prebuild: + family_profiles.append((parts[0], parts[1])) + + registry_path = package_root / "python_profiles.toml" + registry = tomllib.loads(registry_path.read_text(encoding="utf-8")) + selected_profiles = self._projected_selected_profile_names( + projection, + family, + registry, + ) + generic_requirements: list[tuple[str, str]] = [] + for profile, spec in registry.get("profiles", {}).items(): + if profile == "base" or not isinstance(spec, dict): + continue + if spec.get("kind") == "venv" and bool(spec.get("prebuild", True)): + value = spec.get("requirements") + if not isinstance(value, str) or not value.strip(): + raise CiError( + f"projected Python profile {profile!r} has no requirements" + ) + generic_requirements.append((str(profile), value.strip())) + selected_generic = [ + item for item in generic_requirements if item[0] in selected_profiles + ] + profiles = [*family_profiles, *selected_generic] + if not profiles: + return None + + result: set[str] = set() + for _profile, value in profiles: + relative = Path(value) + if relative.is_absolute() or ".." in relative.parts: + raise CiError(f"projected Python profile has an unsafe requirements path: {value!r}") + path = package_root / relative + if not path.is_file() or not path.resolve().is_relative_to(package_root.resolve()): + raise CiError(f"projected Python profile requirements are missing: {value!r}") + for line_number, raw_line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + match = _EXACT_PROFILE_REQUIREMENT_RE.fullmatch(line) + if match is None or _EXACT_PROFILE_VERSION_RE.fullmatch(match.group(2)) is None: + raise CiError( + f"projected Python profile lock must contain exact pins; " + f"{relative}:{line_number} is {raw_line!r}" + ) + result.add(line) + if len(result) > 256: + raise CiError("projected Python profiles declare more than 256 packages") + return _PythonProfilePlan( + names=tuple(sorted(profile for profile, _requirements in profiles)), + packages=tuple(sorted(result)), + ) + + @staticmethod + def _projected_selected_profile_names( + projection: Path, + family: dict[str, object], + registry: dict[str, object], + ) -> set[str]: + """Return profiles selected by projected family and E2E metadata.""" + selected: set[str] = set() + + raw_defaults = family.get("default_execution_profiles", []) + if not isinstance(raw_defaults, list): + raise CiError("projected default_execution_profiles must be a list") + for raw_default in raw_defaults: + if not isinstance(raw_default, str): + raise CiError("projected default_execution_profiles must contain strings") + parts = [part.strip() for part in raw_default.split("|")] + if len(parts) != 2 or any(not part for part in parts): + raise CiError( + f"invalid projected default_execution_profiles entry: {raw_default!r}" + ) + selected.add(parts[1]) + + def add_profiles(value: object, label: str) -> None: + if value is None: + return + if not isinstance(value, dict): + raise CiError(f"projected {label} must be an object") + for profile in value.values(): + if not isinstance(profile, str) or not profile.strip(): + raise CiError(f"projected {label} must select non-empty profiles") + selected.add(profile.strip()) + + e2e_root = projection / "tests/e2e/models" + for manifest_path in sorted(e2e_root.glob("*/manifests/*.json")): + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise CiError(f"projected E2E manifest must be an object: {manifest_path}") + index_path = manifest_path.parent.parent / "MODEL.toml" + e2e_index = ( + tomllib.loads(index_path.read_text(encoding="utf-8")) + if index_path.is_file() + else {} + ) + e2e_defaults = e2e_index.get("e2e_defaults", {}) + if not isinstance(e2e_defaults, dict): + raise CiError(f"projected E2E defaults must be an object: {index_path}") + testcases = manifest.get("testcases", []) + if not isinstance(testcases, list): + raise CiError(f"projected E2E testcases must be a list: {manifest_path}") + cases = [manifest] + for testcase in testcases: + if not isinstance(testcase, dict): + raise CiError(f"projected E2E testcase must be an object: {manifest_path}") + cases.append({**manifest, **testcase}) + for case in cases: + add_profiles(case.get("execution_profiles"), "execution_profiles") + task_strategy = case.get("task_strategy") + task_defaults = ( + e2e_defaults.get(task_strategy, {}) + if isinstance(task_strategy, str) + else {} + ) + if not isinstance(task_defaults, dict): + raise CiError( + f"projected E2E defaults for {task_strategy!r} must be an object" + ) + for section_name, selector_field in ( + ("runtime_strategy_defaults", "runtime_strategy"), + ("reference_backend_defaults", "reference_backend"), + ): + selector = case.get(selector_field, task_defaults.get(selector_field)) + if not isinstance(selector, str) or not selector.strip(): + continue + section = registry.get(section_name, {}) + if not isinstance(section, dict): + raise CiError(f"projected {section_name} must be an object") + add_profiles( + section.get(selector.strip()), + f"{section_name}.{selector.strip()}", + ) + return selected + + def _download_python_profile_packages( + self, + package_dir: Path, + image: str, + packages: tuple[str, ...], + ) -> None: + """Download wheels online without executing contributor-controlled package code.""" + assert self.artifacts_dir is not None + name = self._base_container_name() + "-python-profile-download" + self.container_name = name + self.context.run(["docker", "rm", "-f", name], check=False, capture_output=True) + command = [ + "timeout", + "--kill-after=1m", + "45m", + "docker", + "run", + "--rm", + "--name", + name, + *self._job_labels(), + "--read-only", + "--network", + "bridge", + "--runtime", + "runc", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--ipc", + "private", + "--pids-limit", + "128", + "--memory", + "4g", + "--cpus", + "2", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--mount", + f"type=bind,src={package_dir},dst={PROFILE_PACKAGES_ROOT}", + "--tmpfs", + "/tmp:rw,exec,nosuid,nodev,size=2g", + "--workdir", + "/tmp", + "-e", + "HOME=/tmp", + "-e", + "USER=trtmc-ci", + "-e", + "LOGNAME=trtmc-ci", + "-e", + "NVIDIA_VISIBLE_DEVICES=void", + "-e", + "CUDA_VISIBLE_DEVICES=", + "-e", + "PIP_CONFIG_FILE=/dev/null", + "-e", + "PIP_EXTRA_INDEX_URL=", + image, + "/opt/venv/bin/python", + "/opt/trtmc-profile-downloader.py", + PROFILE_PACKAGES_ROOT, + *packages, + ] + result = self._run_logged( + command, + self.artifacts_dir / "python-profile-download.log", + max_output_chars=2_000_000, + ) + if result: + raise CiError( + f"Python profile package download failed for {self.request.model} " + f"(exit {result})" + ) + def _job_labels(self) -> list[str]: return [ "--label", @@ -622,6 +1004,7 @@ def _run_proof_container( image: str, selection: ModelProofSelection, validation_dir: Path | None, + python_profiles: Path, ) -> None: assert self.lease and self.artifacts_dir is not None and self.lease.gpu_id is not None name = self._base_container_name() @@ -638,6 +1021,11 @@ def _run_proof_container( f"type=bind,src={self.artifacts_dir},dst=/artifacts", "--mount", f"type=bind,src={private_hub},dst=/hf-cache/hub", + "--mount", + ( + f"type=bind,src={python_profiles},dst={PREPARED_PROFILE_ROOT}," + "readonly" + ), ] if validation_dir is not None: mounts.extend( @@ -726,7 +1114,10 @@ def _proof_environment( "TORCHINDUCTOR_CACHE_DIR": "/work/torch-cache", "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", + "PIP_NO_INDEX": "1", "PYTHONHASHSEED": "0", + "TRTMC_PYTHON_PROFILE_PREBUILT_ONLY": "1", + "TRTMC_PYTHON_PROFILE_ROOT": PREPARED_PROFILE_ROOT, "TRTMC_MODEL_PLUGIN_STRICT": "1", "TRTMC_MODEL_PROOF_GPU_ID": str(self.lease.gpu_id), "TRTMC_MODEL_PROOF_GPU_SLOT_IDS": slots, @@ -811,7 +1202,13 @@ def _reclaim_orphans(self) -> None: if container in remaining: raise CiError(f"could not remove orphaned model-proof container {container}") - def _run_logged(self, command: list[object], path: Path) -> int: + def _run_logged( + self, + command: list[object], + path: Path, + *, + max_output_chars: int | None = None, + ) -> int: with path.open("w", encoding="utf-8") as output: process = subprocess.Popen( [str(item) for item in command], @@ -822,9 +1219,22 @@ def _run_logged(self, command: list[object], path: Path) -> int: stderr=subprocess.STDOUT, ) assert process.stdout is not None - for line in process.stdout: - print(line, end="") - output.write(line) + written = 0 + while chunk := process.stdout.read(8192): + if max_output_chars is not None and written + len(chunk) > max_output_chars: + remaining = max(0, max_output_chars - written) + if remaining: + print(chunk[:remaining], end="") + output.write(chunk[:remaining]) + message = "\nERROR: subprocess output limit exceeded\n" + print(message, end="") + output.write(message) + process.kill() + process.wait() + return 125 + print(chunk, end="") + output.write(chunk) + written += len(chunk) return process.wait() def _record_host_error(self, error: BaseException) -> None: diff --git a/tools/ci/profile_downloader.py b/tools/ci/profile_downloader.py new file mode 100644 index 0000000000..c5b9a65338 --- /dev/null +++ b/tools/ci/profile_downloader.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Download exact public PyPI artifacts without executing package source code. + +Boundary: online artifact retrieval only; profile installation and verification are offline. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import urllib.parse +import urllib.request + + +MAX_PACKAGE_COUNT = 256 +MAX_METADATA_BYTES = 4 * 1024 * 1024 +MAX_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024 +MAX_TOTAL_BYTES = 16 * 1024 * 1024 * 1024 + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _response_host(response, expected: str, label: str) -> None: + final_url = urllib.parse.urlparse(response.geturl()) + if final_url.scheme != "https" or final_url.hostname != expected: + raise RuntimeError(f"{label} redirected to an untrusted URL") + + +def _download_sdist(requirement: str, destination: Path) -> None: + distribution, separator, version = requirement.partition("==") + if not separator: + raise RuntimeError(f"invalid exact requirement: {requirement}") + distribution = distribution.partition("[")[0] + endpoint = ( + "https://pypi.org/pypi/" + + urllib.parse.quote(distribution, safe="") + + "/" + + urllib.parse.quote(version, safe="") + + "/json" + ) + request = urllib.request.Request( + endpoint, + headers={"User-Agent": "trtmc-profile-preparer/1"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + _response_host(response, "pypi.org", f"PyPI metadata for {requirement}") + payload_bytes = response.read(MAX_METADATA_BYTES + 1) + if len(payload_bytes) > MAX_METADATA_BYTES: + raise RuntimeError(f"PyPI metadata is unexpectedly large for {requirement}") + payload = json.loads(payload_bytes) + candidates = [ + item for item in payload.get("urls", []) if item.get("packagetype") == "sdist" + ] + if len(candidates) != 1: + raise RuntimeError(f"no unique source distribution is available for {requirement}") + record = candidates[0] + filename = record.get("filename") + url = record.get("url") + expected = record.get("digests", {}).get("sha256") + if ( + not isinstance(filename, str) + or Path(filename).name != filename + or not isinstance(url, str) + or not isinstance(expected, str) + or len(expected) != 64 + ): + raise RuntimeError(f"PyPI returned an invalid source record for {requirement}") + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https" or parsed.hostname != "files.pythonhosted.org": + raise RuntimeError(f"PyPI returned an untrusted source URL for {requirement}") + target = destination / filename + if target.is_file() and _sha256(target) == expected: + return + temporary = destination / f".{filename}.{os.getpid()}.tmp" + artifact_request = urllib.request.Request( + url, + headers={"User-Agent": "trtmc-profile-preparer/1"}, + ) + digest = hashlib.sha256() + size = 0 + try: + with urllib.request.urlopen(artifact_request, timeout=300) as response, temporary.open( + "wb" + ) as output: + _response_host(response, "files.pythonhosted.org", requirement) + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + size += len(chunk) + if size > MAX_ARTIFACT_BYTES: + raise RuntimeError(f"source distribution is too large for {requirement}") + digest.update(chunk) + output.write(chunk) + if digest.hexdigest() != expected: + raise RuntimeError(f"source distribution digest mismatch for {requirement}") + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + + +def download_packages(destination: Path, requirements: list[str]) -> None: + if not requirements or len(requirements) > MAX_PACKAGE_COUNT: + raise ValueError(f"profile package count must be between 1 and {MAX_PACKAGE_COUNT}") + destination.mkdir(parents=True, exist_ok=True) + for requirement in requirements: + result = subprocess.run( + [ + sys.executable, + "-m", + "pip", + "download", + "--disable-pip-version-check", + "--no-cache-dir", + "--no-deps", + "--only-binary=:all:", + "--index-url", + "https://pypi.org/simple", + "--dest", + str(destination), + requirement, + ], + check=False, + ) + if result.returncode: + _download_sdist(requirement, destination) + total_bytes = sum( + path.stat().st_size + for path in destination.iterdir() + if path.is_file() and not path.is_symlink() + ) + if total_bytes > MAX_TOTAL_BYTES: + raise RuntimeError("prepared profile packages exceed the 16 GiB run limit") + + +def main(arguments: list[str] | None = None) -> int: + values = list(sys.argv[1:] if arguments is None else arguments) + if len(values) < 2: + raise SystemExit("usage: profile_downloader.py DESTINATION NAME==VERSION [...]") + download_packages(Path(values[0]), values[1:]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/model_ci.py b/tools/model_ci.py index 602dbe37ef..8a40672275 100644 --- a/tools/model_ci.py +++ b/tools/model_ci.py @@ -55,6 +55,7 @@ PLATFORM_PROJECTION_EXACT = frozenset( { ".clang-format", + ".github/scripts/build-python-profiles.py", ".github/scripts/write-model-proof-fallback-report.py", "ASSET_LICENSES.md", "CMakeLists.txt", diff --git a/website/docs/extend/add-model-family.md b/website/docs/extend/add-model-family.md index e0b9d7f564..4d6e789b93 100644 --- a/website/docs/extend/add-model-family.md +++ b/website/docs/extend/add-model-family.md @@ -103,6 +103,41 @@ policy, and optional debug hooks in this family package. The old repository-root `graph_ops.py`, `graph_blocks.py`, and `standard_decoder_builder.py` ownership model has been retired. +### Optional Python execution profile + +If build or reference code needs Python packages that conflict with the common +environment, keep the declaration and exact pins in the owning family: + +```toml +python_profile_specs = [ + "example_reference|families/example/python_profile_requirements/reference.lock.txt|families/example/python_profile_verify.py|true|true", +] +default_execution_profiles = [ + "reference|example_reference", +] +``` + +The fields are `name|requirements|verifier|system_site_packages|prebuild`. +`prebuild=true` means CI prepares the profile before a network-disabled proof; +it does not bake the profile into the shared base image or change the base +runtime fingerprint. Requirements must be exact public PyPI `name==version` +pins. CI downloads their artifacts with the reviewed base-image downloader, +then installs, source-builds, and verifies them in a separate offline container. + +When an sdist must disable its own network-wheel lookup, declare only the +package build setting in the family descriptor: + +```toml +python_profile_build_environment = [ + "example_reference|PACKAGE_FORCE_BUILD|TRUE", +] +``` + +This surface is for package build switches, not credentials, package indexes, +runtime configuration, or system dependencies. A new APT, CUDA, compiler, or +system-library requirement still changes the reviewed base runtime and needs +maintainer qualification. + ### Optional split decoder contract Opt into separate prefill/decode engines only when the family builder and