ci(profiles): decouple family envs from base image

.dockerignore

@@ -1,25 +1,9 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# The repository Dockerfile copies only the declarative Python profile source
-# and its image-build helper. Keep the daemon context stable and small even on
-# long-lived self-hosted runners with large build/test artifacts.
+# The repository Dockerfile installs the stable base toolchain and copies only
+# its model-agnostic package downloader. Family profiles are prepared per proof.
 *
-!python/
-!python/tensorrt_model_connect/
-!python/tensorrt_model_connect/__init__.py
-!python/tensorrt_model_connect/python_profiles.py
-!python/tensorrt_model_connect/python_profiles.toml
-!python/tensorrt_model_connect/families/
-!python/tensorrt_model_connect/families/__init__.py
-!python/tensorrt_model_connect/families/*/
-!python/tensorrt_model_connect/families/*/MODEL.toml
-!python/tensorrt_model_connect/families/*/python_profile_requirements/
-!python/tensorrt_model_connect/families/*/python_profile_requirements/*.lock.txt
-!python/tensorrt_model_connect/families/*/python_profile_verify.py
-!.github/
-!.github/scripts/
-!.github/scripts/build-python-profiles.py
-
-**/__pycache__/
-**/*.py[cod]
+!tools/
+!tools/ci/
+!tools/ci/profile_downloader.py

.github/scripts/build-python-profiles.py

@@ -1,12 +1,12 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-"""Materialize every declared Python profile during the CI image build."""
+"""Prepare exact-pinned Python profiles before network-disabled execution."""
 
 from __future__ import annotations
 
+import argparse
 import importlib.util
-import json
 import os
 import sys
 import types
@@ -39,33 +39,34 @@ def _load_profile_api():
     return module
 
 
-def main() -> None:
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--profile", action="append", default=[])
+    args = parser.parse_args(argv)
     profile_api = _load_profile_api()
-    names = profile_api.prebuilt_python_profile_names(
+    available = profile_api.prebuilt_python_profile_names(
         profile_api.load_python_profile_registry()
     )
-    if not names:
+    if not available:
         raise SystemExit("no prebuilt Python profiles were declared")
+    requested = tuple(args.profile)
+    if requested:
+        if len(set(requested)) != len(requested):
+            raise SystemExit("requested Python profiles must be unique")
+        unknown = sorted(set(requested) - set(available))
+        if unknown:
+            raise SystemExit("requested Python profiles are not prebuilt: " + ",".join(unknown))
+        names = tuple(sorted(requested))
+    else:
+        names = available
 
     base_python = os.environ.get("TRTMC_BASE_PYTHON", "/opt/venv/bin/python")
-    resolved: dict[str, dict[str, str]] = {}
     for name in names:
         python = profile_api.resolve_profile_python(name, base_python)
         ready = Path(python).parent.parent / ".ready"
         if not ready.is_file():
             raise SystemExit(f"profile {name!r} was not marked ready: {ready}")
-        resolved[name] = {"python": python, "ready": str(ready)}
-
-    manifest = {
-        "schema_version": 1,
-        "profiles": resolved,
-    }
-    root = profile_api.profile_root()
-    (root / ".image-ready.json").write_text(
-        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
-        encoding="utf-8",
-    )
-    print("prebuilt_python_profiles=" + ",".join(names))
+    print("prepared_python_profiles=" + ",".join(names))
 
 
 if __name__ == "__main__":

.github/scripts/write-model-proof-fallback-report.py

@@ -9,20 +9,25 @@
 import argparse
 import html
 import json
+import os
+import stat
 from pathlib import Path
 from typing import Sequence
 
 
 _DIAGNOSTIC_FILES = (
     "host-error.log",
     "ci-image.log",
+    "python-profiles-prepare.log",
+    "python-profile-download.log",
     "console.log",
     "projection.stderr.log",
     "projection.json",
     "configure.log",
     "build.log",
 )
 _MAX_DIAGNOSTIC_CHARS = 16_000
+_MAX_DIAGNOSTIC_BYTES = _MAX_DIAGNOSTIC_CHARS * 4
 
 
 def _load_json(path: Path) -> dict[str, object]:
@@ -37,14 +42,29 @@ def _diagnostics(root: Path) -> list[tuple[str, str]]:
     excerpts: list[tuple[str, str]] = []
     for filename in _DIAGNOSTIC_FILES:
         path = root / filename
-        if not path.is_file():
+        try:
+            descriptor = os.open(
+                path,
+                os.O_RDONLY
+                | getattr(os, "O_NONBLOCK", 0)
+                | getattr(os, "O_NOFOLLOW", 0),
+            )
+        except OSError:
             continue
         try:
-            text = path.read_text(encoding="utf-8", errors="replace")
+            metadata = os.fstat(descriptor)
+            if not stat.S_ISREG(metadata.st_mode):
+                continue
+            offset = max(0, metadata.st_size - _MAX_DIAGNOSTIC_BYTES)
+            os.lseek(descriptor, offset, os.SEEK_SET)
+            payload = os.read(descriptor, _MAX_DIAGNOSTIC_BYTES)
         except OSError:
             continue
+        finally:
+            os.close(descriptor)
+        text = payload.decode("utf-8", errors="replace")[-_MAX_DIAGNOSTIC_CHARS:]
         if text.strip():
-            excerpts.append((filename, text[-_MAX_DIAGNOSTIC_CHARS:]))
+            excerpts.append((filename, text))
     return excerpts
 
 

Dockerfile

@@ -135,44 +135,20 @@ RUN pip install --force-reinstall \
 # reference inference on the system cuBLAS instead of pip-installed CUDA libs.
 ENV LD_PRELOAD=/usr/local/cuda/lib64/libcublas.so.13
 
+# This model-agnostic downloader is part of the reviewed base runtime. It may
+# fetch exact public PyPI artifacts, but never imports or builds package code.
+COPY tools/ci/profile_downloader.py /opt/trtmc-profile-downloader.py
+
 # Coverage tooling verification (run inside container):
 #   python3 -m coverage --version && pytest --version && \
 #   python3 -m pytest --help | grep -- '--cov' && \
 #   gcovr --version && lcov --version && genhtml --version
 
-# Build every declarative Python execution profile while network access is
-# available. Family-owned declarations, lock files, and verification scripts
-# are the package source of truth; python_profiles.py rejects non-exact pins and
-# verifies every installed distribution before marking a profile ready.
-FROM ci-common-base AS python-profile-builder
-
-ENV TRTMC_PYTHON_PROFILE_ROOT=/opt/trtmc-python-profiles
-# sphn publishes no aarch64 wheel. Keep its Rust build toolchain in this
-# throwaway builder stage; the final development stage receives only the
-# verified profile.
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends cargo rustc && \
-    rm -rf /var/lib/apt/lists/* && \
-    pip install "maturin==1.14.1"
-# Avoid compiling profile-local CUDA extensions for every architecture known
-# to a GPU-less Docker build. Keep 10.0 as the GB300 target.
-COPY python/tensorrt_model_connect /opt/trtmc-profile-source/tensorrt_model_connect
-COPY .github/scripts/build-python-profiles.py /opt/trtmc-build-python-profiles.py
-RUN python3 /opt/trtmc-build-python-profiles.py \
-    && chmod -R a+rX /opt/trtmc-python-profiles
-
-# Do not retain the full builder source tree in the development image. Only the
-# verified virtual environments cross the stage boundary, so sibling model
-# implementations cannot satisfy imports in an isolated source projection.
+# Keep the reusable runtime independent of family-owned Python environments.
+# Online CI preparation materializes the selected family's exact-pinned
+# profiles before a network-disabled proof and mounts them read-only there.
 FROM ci-common-base AS ci-common
 
-COPY --from=python-profile-builder \
-    /opt/trtmc-python-profiles /opt/trtmc-python-profiles
-ENV TRTMC_PYTHON_PROFILE_ROOT=/opt/trtmc-python-profiles
-# Execution-profile environments are part of the dev-image contract. Rebuild
-# the image after changing their lock or verification files.
-ENV TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1
-
 # Keep the reusable common layer independent of every TensorRT release. The
 # version overlay below is the only stage allowed to add bindings, headers, or
 # native runtime libraries.

python/tensorrt_model_connect/families/nemotron_h/MODEL.toml

@@ -7,6 +7,10 @@ module = "plugin"
 python_profile_specs = [
   "nemotron_h_reference|families/nemotron_h/python_profile_requirements/nemotron_h_reference.lock.txt|families/nemotron_h/python_profile_verify.py|true",
 ]
+python_profile_build_environment = [
+  "nemotron_h_reference|CAUSAL_CONV1D_FORCE_BUILD|TRUE",
+  "nemotron_h_reference|MAMBA_FORCE_BUILD|TRUE",
+]
 default_execution_profiles = [
   "reference|nemotron_h_reference",
 ]

python/tensorrt_model_connect/python_profiles.py

@@ -45,6 +45,25 @@
     re.IGNORECASE,
 )
 _PROFILE_NAME_RE = re.compile(r"[a-z][a-z0-9_]*")
+_BUILD_ENVIRONMENT_NAME_RE = re.compile(r"[A-Z][A-Z0-9_]*")
+_FORBIDDEN_BUILD_ENVIRONMENT_NAMES = {
+    "HOME",
+    "LD_LIBRARY_PATH",
+    "LD_PRELOAD",
+    "PATH",
+    "PYTHONHOME",
+    "PYTHONPATH",
+}
+_FORBIDDEN_BUILD_ENVIRONMENT_PREFIXES = (
+    "AWS_",
+    "AZURE_",
+    "GIT_",
+    "GOOGLE_",
+    "NVIDIA_",
+    "PIP_",
+    "SSH_",
+    "TRTMC_",
+)
 _REGISTRY_KEYS = {
     "version",
     "profiles",
@@ -55,6 +74,7 @@
 _VENV_PROFILE_KEYS = {
     "kind",
     "prebuild",
+    "build_environment",
     "requirements",
     "system_site_packages",
     "verification_script",
@@ -153,6 +173,7 @@ def family_python_profile_specs() -> dict[str, dict[str, object]]:
         with manifest.open("rb") as stream:
             raw = tomllib.load(stream)
         family_id = raw.get("id") or raw.get("plugin") or manifest.parent.name
+        family_profile_names: set[str] = set()
         raw_specs = raw.get("python_profile_specs", [])
         if not isinstance(raw_specs, list):
             raise ValueError(
@@ -192,6 +213,49 @@ def family_python_profile_specs() -> dict[str, dict[str, object]]:
                 "system_site_packages": system_site_packages,
                 "prebuild": prebuild,
             }
+            family_profile_names.add(name)
+        raw_build_environment = raw.get("python_profile_build_environment", [])
+        if not isinstance(raw_build_environment, list):
+            raise ValueError(
+                f"python_profile_build_environment for family {family_id} must be a list"
+            )
+        for entry in raw_build_environment:
+            if not isinstance(entry, str):
+                raise ValueError(
+                    f"python_profile_build_environment for family {family_id} "
+                    "must contain strings"
+                )
+            parts = [part.strip() for part in entry.split("|", 2)]
+            if len(parts) != 3 or any(not part for part in parts):
+                raise ValueError(
+                    f"Invalid python_profile_build_environment entry {entry!r} "
+                    f"for family {family_id}; expected 'profile|NAME|value'"
+                )
+            profile, name, value = parts
+            if profile not in family_profile_names:
+                raise ValueError(
+                    f"python_profile_build_environment selects undeclared profile "
+                    f"{profile!r} for family {family_id}"
+                )
+            if (
+                _BUILD_ENVIRONMENT_NAME_RE.fullmatch(name) is None
+                or name in _FORBIDDEN_BUILD_ENVIRONMENT_NAMES
+                or name.startswith(_FORBIDDEN_BUILD_ENVIRONMENT_PREFIXES)
+            ):
+                raise ValueError(
+                    f"Python profile {profile!r} has unsafe build environment name {name!r}"
+                )
+            if len(value) > 1024 or "\x00" in value or "\n" in value or "\r" in value:
+                raise ValueError(
+                    f"Python profile {profile!r} has an unsafe build environment value"
+                )
+            build_environment = dict(profiles[profile].get("build_environment", {}))
+            if name in build_environment:
+                raise ValueError(
+                    f"Python profile {profile!r} declares build environment {name!r} twice"
+                )
+            build_environment[name] = value
+            profiles[profile]["build_environment"] = build_environment
     return profiles
 
 
@@ -260,6 +324,23 @@ def _validate_python_profile_registry(registry: Mapping[str, Any]) -> None:
                 raise ValueError(
                     f"Execution profile {name!r} field {field} must be a bool"
                 )
+        build_environment = raw_spec.get("build_environment", {})
+        if not isinstance(build_environment, Mapping) or any(
+            not isinstance(name, str)
+            or _BUILD_ENVIRONMENT_NAME_RE.fullmatch(name) is None
+            or name in _FORBIDDEN_BUILD_ENVIRONMENT_NAMES
+            or name.startswith(_FORBIDDEN_BUILD_ENVIRONMENT_PREFIXES)
+            or not isinstance(value, str)
+            or not value
+            or len(value) > 1024
+            or "\x00" in value
+            or "\n" in value
+            or "\r" in value
+            for name, value in build_environment.items()
+        ):
+            raise ValueError(
+                f"Execution profile {name!r} build_environment must contain safe strings"
+            )
         requirements = raw_spec.get("requirements")
         if type(requirements) is not str:
             raise ValueError(
@@ -334,7 +415,7 @@ def _validate_python_profile_registry(registry: Mapping[str, Any]) -> None:
 def prebuilt_python_profile_names(
     registry: Mapping[str, Any] | None = None,
 ) -> tuple[str, ...]:
-    """Return non-default profiles that belong in the shared CI image."""
+    """Return non-default profiles prepared before network-disabled execution."""
     selected = (
         registry if registry is not None else load_python_profile_registry()
     )
@@ -669,6 +750,10 @@ def _materialize_venv_profile(
             )
         verification_script = _read_package_text(verification_script_file).strip()
     system_site_packages = bool(spec.get("system_site_packages", True))
+    build_environment = {
+        str(name): str(value)
+        for name, value in dict(spec.get("build_environment", {})).items()
+    }
 
     hash_input = "\n".join(
         [
@@ -678,6 +763,7 @@ def _materialize_venv_profile(
             requirements_text,
             verification_script,
             f"system_site_packages={int(system_site_packages)}",
+            json.dumps(build_environment, separators=(",", ":"), sort_keys=True),
         ]
     ).encode("utf-8")
     profile_hash = hashlib.sha256(hash_input).hexdigest()[:12]
@@ -688,15 +774,14 @@ def _materialize_venv_profile(
     ready_path = env_dir / ".ready"
     lock_path = root / f"{profile_name}-{profile_hash}.lock"
 
-    # Model-proof containers mount the source read-only and disable networking.
-    # A matching image-baked profile therefore needs no writable lock or cache.
+    # Network-disabled proofs mount a separately prepared profile root read-only.
     if ready_path.is_file() and python_path.is_file():
         return str(python_path.absolute())
     if _prebuilt_only():
         raise RuntimeError(
             f"Execution profile {profile_name!r} is not prebuilt for this source "
-            f"at {env_dir}. The CI image is stale or incomplete; rebuild it from "
-            "the current Dockerfile and declarative profile locks."
+            f"at {env_dir}. Prepare the declared profiles before entering the "
+            "network-disabled execution lane."
         )
 
     root.mkdir(parents=True, exist_ok=True)
@@ -726,6 +811,8 @@ def _materialize_venv_profile(
                 _write_base_site_packages_overlay(base_python, str(tmp_python))
 
             if requirements_text.strip():
+                install_environment = _profile_install_environment()
+                install_environment.update(build_environment)
                 _run_profile_command(
                     [
                         str(tmp_python),
@@ -741,7 +828,7 @@ def _materialize_venv_profile(
                     ],
                     description=f"install Python profile {profile_name!r}",
                     timeout=_PROFILE_INSTALL_TIMEOUT_SECONDS,
-                    env=_profile_install_environment(),
+                    env=install_environment,
                 )
 
             _verify_exact_requirements(