feat: append builder info to detect builds made with uncommitted changes in the builder

docs/source/builder-cli.md

@@ -244,6 +244,10 @@ Generate CMake files for a kernel extension build
 
 * `-f`, `--force` — Force-overwrite existing files
 * `--unique-id <UNIQUE_ID>` — This is an optional unique identifier that is suffixed to the kernel name to avoid name collisions. (e.g. Git SHA)
+* `--kernel-sha <KERNEL_SHA>` — Full commit SHA of the kernel source, recorded in the build metadata. When absent, it is detected from the kernel's git repository (used by Nix builds where the source has no `.git`)
+* `--kernel-dirty` — Mark the kernel source as having uncommitted changes in the build metadata. Only meaningful together with `--kernel-sha`
+* `--kernel-builder-sha <KERNEL_BUILDER_SHA>` — Full commit SHA of the `kernel-builder` source, recorded in the build metadata. When absent, the SHA baked in at compile time is used (Nix builds pass this since the sandbox has no `.git`)
+* `--kernel-builder-dirty` — Mark `kernel-builder` as having uncommitted changes in the build metadata. Only meaningful together with `--kernel-builder-sha`
 
 
 

docs/source/kernel-requirements.md

@@ -81,6 +81,26 @@ metadata. Currently the following top-level keys are supported:
 - `digest` (`Digest`, required): hash digest of the kernel files.
 - `python-depends` (`list[str]`, optional): list of Python dependencies
   from a curated set of Python dependencies.
+- `build-info` (`dict`, optional): provenance of the build, used to flag
+  non-reproducible (dirty) builds. It contains two optional sub-objects:
+  - `kernel-builder`: the `kernel-builder` that produced the build, with its
+    `version` (`str`), the `sha` (`str`) of the `kernel-builder` source it was
+    built from (when known), and a `dirty` (`bool`) flag that is `true` when
+    `kernel-builder` was built from a source tree with uncommitted changes.
+  - `kernel`: the kernel source that was built, with its commit `sha` (`str`)
+    and a `dirty` (`bool`) flag that is `true` when the kernel source had
+    uncommitted changes.
+
+  When either `dirty` flag is set, the kernel was built from uncommitted
+  sources and cannot be reliably reproduced. `kernels` warns when loading such
+  a build, `kernel-builder` warns when uploading it, and a warning banner is
+  added to the Hub repository's README.
+
+  > **Note:** For Nix builds, dirtiness follows Nix's flake tree status, which
+  > also counts **untracked** files (including an uncommitted `flake.lock`).
+  > Commit your `flake.lock` (and avoid stray untracked files) so that clean
+  > builds are not flagged as dirty. Local `create-pyproject` runs only
+  > consider changes to tracked files.
 
 Example `metadata.json`:
 

flake.nix

@@ -37,6 +37,14 @@
         ;
       inherit (import ./nix-builder/lib/cache.nix) mkForCache;
 
+      # Git provenance of `kernel-builder` itself, recorded in the build
+      # metadata of kernels it builds. `self` here is the `kernel-builder`
+      # flake (as resolved in the consuming kernel's lock). It is captured
+      # here because the `self` argument of `genKernelFlakeOutputs` below
+      # shadows it with the *kernel's* `self`.
+      builderRev = self.rev or self.dirtyRev or null;
+      builderDirty = !(self ? rev);
+
       systems = with flake-utils.lib.system; [
         aarch64-darwin
         aarch64-linux
@@ -109,6 +117,8 @@
                 path
                 rev
                 self
+                builderRev
+                builderDirty
                 doGetKernelCheck
                 pythonCheckInputs
                 pythonNativeCheckInputs

kernel-builder/build.rs

@@ -1,3 +1,52 @@
+use std::process::Command;
+
 fn main() {
     minijinja_embed::embed_templates!("src/pyproject/templates");
+    emit_git_info();
+}
+
+fn emit_git_info() {
+    let sha = env_var("KERNEL_BUILDER_GIT_SHA").or_else(|| git_output(&["rev-parse", "HEAD"]));
+    if let Some(sha) = sha {
+        println!("cargo:rustc-env=KERNEL_BUILDER_GIT_SHA={sha}");
+    }
+
+    let dirty = match env_var("KERNEL_BUILDER_GIT_DIRTY") {
+        Some(value) => is_truthy(&value),
+        // Only consider tracked files; untracked files (e.g. generated build
+        // artifacts) should not mark the build as dirty.
+        None => git_output(&["status", "--porcelain", "--untracked-files=no"])
+            .map(|out| !out.is_empty())
+            .unwrap_or(false),
+    };
+    println!(
+        "cargo:rustc-env=KERNEL_BUILDER_GIT_DIRTY={}",
+        if dirty { "1" } else { "0" }
+    );
+
+    println!("cargo:rerun-if-env-changed=KERNEL_BUILDER_GIT_SHA");
+    println!("cargo:rerun-if-env-changed=KERNEL_BUILDER_GIT_DIRTY");
+    println!("cargo:rerun-if-changed=../.git/HEAD");
+    println!("cargo:rerun-if-changed=../.git/index");
+}
+
+fn env_var(name: &str) -> Option<String> {
+    std::env::var(name).ok().filter(|s| !s.is_empty())
+}
+
+fn is_truthy(value: &str) -> bool {
+    value == "1" || value.eq_ignore_ascii_case("true")
+}
+
+fn git_output(args: &[&str]) -> Option<String> {
+    let output = Command::new("git").args(args).output().ok()?;
+    if !output.status.success() {
+        return None;
+    }
+    let s = String::from_utf8_lossy(&output.stdout).trim().to_string();
+    if s.is_empty() {
+        None
+    } else {
+        Some(s)
+    }
 }

kernel-builder/src/main.rs

@@ -174,6 +174,28 @@ enum Commands {
         /// kernel name to avoid name collisions. (e.g. Git SHA)
         #[arg(long)]
         unique_id: Option<String>,
+
+        /// Full commit SHA of the kernel source, recorded in the build
+        /// metadata. When absent, it is detected from the kernel's git
+        /// repository (used by Nix builds where the source has no `.git`).
+        #[arg(long)]
+        kernel_sha: Option<String>,
+
+        /// Mark the kernel source as having uncommitted changes in the build
+        /// metadata. Only meaningful together with `--kernel-sha`.
+        #[arg(long)]
+        kernel_dirty: bool,
+
+        /// Full commit SHA of the `kernel-builder` source, recorded in the
+        /// build metadata. When absent, the SHA baked in at compile time is
+        /// used (Nix builds pass this since the sandbox has no `.git`).
+        #[arg(long)]
+        kernel_builder_sha: Option<String>,
+
+        /// Mark `kernel-builder` as having uncommitted changes in the build
+        /// metadata. Only meaningful together with `--kernel-builder-sha`.
+        #[arg(long)]
+        kernel_builder_dirty: bool,
     },
 
     /// Spawn a kernel development shell.
@@ -365,7 +387,20 @@ fn main() -> Result<()> {
             force,
             target_dir,
             unique_id,
-        } => create_pyproject(kernel_dir, target_dir, force, unique_id),
+            kernel_sha,
+            kernel_dirty,
+            kernel_builder_sha,
+            kernel_builder_dirty,
+        } => create_pyproject(
+            kernel_dir,
+            target_dir,
+            force,
+            unique_id,
+            kernel_sha,
+            kernel_dirty,
+            kernel_builder_sha,
+            kernel_builder_dirty,
+        ),
         Commands::Devshell {
             kernel_dir,
             variant,

kernel-builder/src/pyproject/common.rs

@@ -4,7 +4,7 @@ use eyre::Result;
 use itertools::Itertools;
 
 use kernels_data::config::{Backend, General};
-use kernels_data::metadata::{BackendInfo, Metadata};
+use kernels_data::metadata::{BackendInfo, BuildInfo, KernelBuilderInfo, Metadata};
 
 use crate::pyproject::ops_identifier::KernelIdentifier;
 use crate::pyproject::FileSet;
@@ -20,11 +20,30 @@ pub fn write_compat_py(file_set: &mut FileSet) -> Result<()> {
     Ok(())
 }
 
+fn kernel_builder_info() -> KernelBuilderInfo {
+    KernelBuilderInfo {
+        version: env!("CARGO_PKG_VERSION").to_owned(),
+        sha: option_env!("KERNEL_BUILDER_GIT_SHA").map(str::to_owned),
+        dirty: matches!(option_env!("KERNEL_BUILDER_GIT_DIRTY"), Some("1")),
+    }
+}
+
 pub fn write_metadata(
     general: &General,
     kernel_id: &KernelIdentifier,
     file_set: &mut FileSet,
 ) -> Result<()> {
+    // Prefer externally-provided `kernel-builder` provenance (e.g. from Nix),
+    // falling back to the provenance baked in at compile time.
+    let kernel_builder = kernel_id
+        .kernel_builder()
+        .cloned()
+        .unwrap_or_else(kernel_builder_info);
+    let build_info = BuildInfo {
+        kernel_builder: Some(kernel_builder),
+        kernel: kernel_id.git_info().cloned(),
+    };
+
     for backend in &Backend::all() {
         let writer = file_set.entry(format!("metadata-{backend}.json"));
 
@@ -51,6 +70,7 @@ pub fn write_metadata(
                 backend_type: *backend,
             },
             digest: None,
+            build_info: Some(build_info.clone()),
         };
 
         serde_json::to_writer_pretty(writer, &metadata)?;

kernel-builder/src/pyproject/mod.rs

@@ -6,6 +6,7 @@ use std::{
 
 use eyre::{bail, Result};
 use kernels_data::config::{Build, Framework};
+use kernels_data::metadata::{GitInfo, KernelBuilderInfo};
 use minijinja::Environment;
 
 use crate::{
@@ -39,16 +40,38 @@ pub fn create_pyproject_file_set(build: Build, kernel_id: &KernelIdentifier) ->
     Ok(file_set)
 }
 
+#[allow(clippy::too_many_arguments)]
 pub fn create_pyproject(
     kernel_dir: Option<PathBuf>,
     target_dir: Option<PathBuf>,
     force: bool,
     unique_id: Option<String>,
+    kernel_sha: Option<String>,
+    kernel_dirty: bool,
+    kernel_builder_sha: Option<String>,
+    kernel_builder_dirty: bool,
 ) -> Result<()> {
     let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?;
     let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?;
     let build = parse_build(&kernel_dir)?;
-    let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id);
+    let git_override = kernel_sha.map(|sha| GitInfo {
+        sha,
+        dirty: kernel_dirty,
+    });
+    // The version is always that of the running `kernel-builder`; only the
+    // git provenance is supplied externally (e.g. by Nix).
+    let kernel_builder_override = kernel_builder_sha.map(|sha| KernelBuilderInfo {
+        version: env!("CARGO_PKG_VERSION").to_owned(),
+        sha: Some(sha),
+        dirty: kernel_builder_dirty,
+    });
+    let kernel_id = KernelIdentifier::new(
+        &kernel_dir,
+        build.general.name.python_name(),
+        unique_id,
+        git_override,
+        kernel_builder_override,
+    );
     let file_set = create_pyproject_file_set(build, &kernel_id)?;
     file_set.write(&target_dir, force)?;
 
@@ -65,7 +88,14 @@ pub fn clean_pyproject(
     let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?;
     let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?;
     let build = parse_build(&kernel_dir)?;
-    let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id);
+    // Provenance is irrelevant when computing the set of files to clean.
+    let kernel_id = KernelIdentifier::new(
+        &kernel_dir,
+        build.general.name.python_name(),
+        unique_id,
+        None,
+        None,
+    );
 
     let generated_files = create_pyproject_file_set(build, &kernel_id)?.into_names();