Merge branch 'main' into version-readme-cards

Author: sayakpaul

kernel-builder/src/pyproject/templates/torch/noarch/setup.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 
-from typing import Any
-from pathlib import Path
 import shutil
 import sys
+from pathlib import Path
+from typing import Any
 
 from setuptools import setup
 from setuptools.command.build import build
@@ -73,9 +73,10 @@ def build_backend(
             raise FileNotFoundError(f"torch-ext/{module_name} does not exist")
         for item in torch_ext_dir.rglob("*"):
             if item.is_file():
-                if item.suffix == ".pyc" or item.parent.name == "__pycache__":
-                    continue
                 rel_path = item.relative_to(torch_ext_dir)
+                # Skip Python bytecode.
+                if item.suffix == ".pyc" or "__pycache__" in rel_path.parts:
+                    continue
                 dest = variant_dir / rel_path
                 dest.parent.mkdir(parents=True, exist_ok=True)
                 shutil.copy2(item, dest)

kernels-data/src/digest.rs

@@ -30,8 +30,10 @@ impl Digest {
                 // hashes. It is protected by the detached signature.
                 || entry.path().file_name().and_then(|f| f.to_str()) == Some("metadata.json")
                 || entry.path().file_name().and_then(|f| f.to_str()) == Some("metadata.json.sigstore")
-                // Python likes to create .pyc files in __pycache__/ directories.
+                // Exclude Python bytecode, these files are generated by the
+                // interpreter since they are never included in builds or downloads.
                 || entry.path().components().any(|c| c.as_os_str() == "__pycache__")
+                || entry.path().extension().and_then(|e| e.to_str()) == Some("pyc")
             {
                 continue;
             }
@@ -130,15 +132,13 @@ mod tests {
         fs::write(dir.path().join("metadata.json"), b"{}")?;
         fs::write(dir.path().join("metadata.json.sigstore"), b"sig")?;
         fs::write(pycache_dir.join("mod.cpython-311.pyc"), b"bytecode")?;
-        // A .pyc outside __pycache__/ must be included — it was not generated
-        // locally by Python and should be covered by the digest.
         fs::write(dir.path().join("payload.pyc"), b"smuggled")?;
 
         let digest = Digest::hash_variant(DigestAlgorithm::SHA256, dir.path())?;
 
-        assert_eq!(digest.files().len(), 2);
+        assert_eq!(digest.files().len(), 1);
         assert!(digest.files().contains_key("_extension.so"));
-        assert!(digest.files().contains_key("payload.pyc"));
+        assert!(!digest.files().contains_key("payload.pyc"));
         assert!(!digest.files().contains_key("metadata.json"));
         assert!(!digest.files().contains_key("metadata.json.sigstore"));
         assert!(

kernels/src/kernels/lockfile.py

@@ -72,7 +72,10 @@ def get_kernel_locks(repo_id: str, version_spec: int) -> KernelLock:
             if sibling.blob_id is None:
                 raise ValueError(f"Cannot get blob ID for {sibling.rfilename}")
 
-            if sibling.rfilename.endswith(".pyc") or "__pycache__" in sibling.rfilename:
+            # Exclude Python bytecode. If bytecode exists, it is generated
+            # by the interpreter, since we exclude bytecode from builds
+            # and downloads.
+            if sibling.rfilename.endswith(".pyc") or "__pycache__" in sibling.rfilename.split("/"):
                 continue
 
             path = Path(sibling.rfilename)

kernels/src/kernels/utils.py

@@ -36,6 +36,12 @@
 
 KNOWN_BACKENDS = {"cpu", "cuda", "metal", "neuron", "rocm", "xpu", "npu"}
 
+# Exclude patter for bytecode. These are not included in kernel builds,
+# but builds not done using kernel-builder might accidentally include
+# bytcode. So these patterns are used to ensure that they are never
+# downloaded.
+_BYTECODE_IGNORE_PATTERNS = ["*.pyc", "**/__pycache__/**"]
+
 
 def _check_trust_remote_code(repo_id: str, trust_remote_code: bool | list[str]) -> None:
     """Check whether a kernel repository is trusted.
@@ -302,7 +308,7 @@ def install_kernel(
         _validate_variant_dependencies(Path(metadata_path).parent)
 
     allow_patterns = [f"build/{variant.variant_str}/*"]
-    ignore_patterns = ["*.pyc", "**/__pycache__/**"]
+    ignore_patterns = _BYTECODE_IGNORE_PATTERNS
 
     repo_path = Path(
         str(
@@ -368,7 +374,7 @@ def _resolve_local_variant_path(
         )
 
     allow_patterns = [f"build/{variant.variant_str}/*"]
-    ignore_patterns = ["*.pyc", "**/__pycache__/**"]
+    ignore_patterns = _BYTECODE_IGNORE_PATTERNS
     repo_path = Path(
         str(
             api.snapshot_download(
@@ -420,7 +426,7 @@ def install_kernel_all_variants(
                 repo_id,
                 repo_type="kernel",
                 allow_patterns="build/*",
-                ignore_patterns=["*.pyc", "**/__pycache__/**"],
+                ignore_patterns=_BYTECODE_IGNORE_PATTERNS,
                 cache_dir=CACHE_DIR,
                 revision=revision,
                 local_files_only=local_files_only,