Merge branch 'libvirt-image-sync'

Author: paulgessinger

config.libvirt.example.toml

@@ -17,10 +17,22 @@ shrink_ticks = 3
 [backend]
 name = "libvirt-gpu"
 type = "libvirt"
-image_name = "husk-gpu-golden.qcow2"   # golden qcow2 filename in each host's pool dir
 min_ready = 2                          # global warm-pool size across all hosts
 max_total = 2                          # one GPU host, one GPU → 1 (raise as the pool grows)
 
+# Image source — pick ONE:
+#  (a) image_ref: an OCI artifact built by build-images.yml and published to
+#      ghcr. The controller `oras pull`s it once to a local cache and scp's it to
+#      every host pool (named by content digest). Pin a tag or a digest. Editing
+#      this while huskd runs hot-reloads: the new image is staged and idle slots
+#      drain onto it (running jobs finish first), old goldens are GC'd.
+image_ref = "ghcr.io/acts-project/husk-gpu:v1"
+# image_cache_dir = "/var/cache/husk/images"   # optional; default ~/.cache/husk/images
+#
+#  (b) image_name: a qcow2 filename you placed in each host's pool dir yourself
+#      (the manual path — no registry, no auto-sync/drain).
+# image_name = "husk-gpu-golden.qcow2"
+
 # One entry per VM-host. GPU host: list gpu_pci_addresses (one slot per GPU).
 # CPU host: omit them and set max_slots (default 1). Setting both is rejected.
 [[backend.hosts]]

image-pipeline.md

@@ -4,10 +4,16 @@ How husk builds, distributes, and rolls out the golden VM images that back its
 runner slots. Companion to `plan.md` (which deliberately deferred custom images
 in "Phase 4"); this document un-defers that work with a concrete design.
 
-> **Status (2026-06-16):** design agreed, not yet built. The runner/podman stack
-> is currently installed at *cloud-init time* as a spike (`src/husk/cloudinit.py`)
-> and the only image-build artifact today is the manual, single-host
-> `scripts/build-golden-image.sh`. This document describes the target system.
+> **Status (2026-06-16):** Phase A (CI build → ghcr via ORAS) merged. **Phases B
+> + C are now built for the libvirt backend** (`src/husk/image_sync.py` +
+> `LibvirtBackend.sync_images`): the controller pulls the config-pinned
+> `image_ref` (via the pure-Python `oras` client — no CLI dep) to a local cache,
+> scp's it to each host pool by digest, stamps the
+> digest into slot metadata, and drains slots onto a new ref on config
+> hot-reload, GC'ing orphaned goldens. **OpenStack/Glance delivery is still
+> deferred** (OpenStack remains the CPU path; the GPU image ships only to libvirt
+> hosts). The runner/podman stack is still installed at cloud-init time for slots
+> that boot a stock base; a baked golden is the boot-speed optimization.
 
 -----
 
@@ -149,13 +155,18 @@ live per-slot overlay — overwriting it in place corrupts running slots. So:
 
 ## Phasing
 
-- **Phase A — CI image build (this is next; see implementation plan).** Single
-  spec, two variants, built in GitHub CI, pushed to ghcr via ORAS. Output:
-  pullable `husk-base` + `husk-gpu` artifacts. *Does not touch huskd.*
-- **Phase B — delivery & sync.** `image_sync.py`, config ref change, oras pull
-  on the controller, libvirt scp + Glance upload.
-- **Phase C — versioned rollout / drain.** Digest stamping + drain-on-change +
-  GC.
+- **Phase A — CI image build. ✅ DONE.** Single spec, two variants, built in
+  GitHub CI, pushed to ghcr via ORAS. Output: pullable `husk-base` + `husk-gpu`
+  artifacts (`build-images.yml`). *Does not touch huskd.*
+- **Phase B — delivery & sync (libvirt). ✅ DONE.** `image_sync.py` (`oras
+  resolve`+`pull` to a controller cache, content-addressed), `LibvirtBackend.
+  sync_images` scp's the golden to each host pool by digest (idempotent, atomic),
+  driven by the config `image_ref`. *Glance upload still deferred.*
+- **Phase C — versioned rollout / drain (libvirt). ✅ DONE.** Digest stamped into
+  domain metadata; `Slot.image_stale` set when it diverges from the host's current
+  image; the controller drains stale idle slots (rebuild adopts the new golden)
+  and `_gc_goldens` removes unreferenced backing files. `image_ref` is
+  hot-reloadable (per-host overrides remain restart-only).
 
 Each phase is independently useful: A produces artifacts you can place by hand
 (exactly the manual step today, just reproducible); B automates delivery against

images/build.sh

@@ -110,6 +110,17 @@ ARGS=(
   # enabled — cloud-init starts it each cycle after laying the JIT config.
   --run-command 'systemctl --global enable podman.socket'
   --run-command 'mkdir -p /var/lib/systemd/linger && touch /var/lib/systemd/linger/runner'
+
+  # SELinux: the CI build host is non-SELinux Ubuntu, which can't reliably relabel
+  # an EL guest offline — virt-customize's --selinux-relabel (below) only defers by
+  # touching /.autorelabel, and a first boot under *enforcing* then deadlocks:
+  # nothing is labeled, so nothing — including the autorelabel service itself —
+  # can exec, and every unit dies with status=127 (a wedged VM, no serial output).
+  # Boot permissive so exec is never blocked; the deferred autorelabel still runs
+  # and completes. Safe for husk: runners are ephemeral + firewall-isolated, and
+  # cloud-init already sets containers.conf label=false, so guest SELinux is not
+  # load-bearing here.
+  --run-command "sed -i 's/^SELINUX=.*/SELINUX=permissive/' /etc/selinux/config"
 )
 
 # --------------------------------------------------------------- gpu additions

pyproject.toml

@@ -14,7 +14,8 @@ dependencies = [
 [project.optional-dependencies]
 dev = ["pytest", "ruff>=0.15"]
 # libvirt backend — kept optional so the OpenStack/CI path needs no libvirt headers.
-libvirt = ["libvirt-python>=10"]
+# `oras` is the pure-Python OCI client used to pull golden images (no oras CLI dep).
+libvirt = ["libvirt-python>=10", "oras>=0.2"]
 
 [project.scripts]
 huskd = "husk.cli:huskd"

src/husk/config.py

@@ -66,15 +66,22 @@ class HostConfig:
     gpu_pci_addresses: tuple[str, ...] = ()
     max_slots: int | None = None  # CPU host capacity; None → 1 (and GPU forbids it)
     image_name: str | None = None  # per-host override of the backend golden image
+    image_ref: str | None = None  # per-host override of the backend OCI image ref
 
 
 @dataclass(frozen=True)
 class BackendConfig:
     name: str
     type: str
-    image_name: str  # OpenStack image name OR libvirt golden qcow2 (shared field)
     min_ready: int
     max_total: int
+    # Image source. OpenStack uses `image_name` (a Glance image name). libvirt
+    # uses `image_ref` (an OCI artifact ref, e.g. ghcr.io/org/husk-gpu:v1 — synced
+    # to each host by the controller) when set, else `image_name` as a literal
+    # qcow2 filename already present in the host pool (the manual/local path).
+    image_name: str = ""
+    image_ref: str = ""  # libvirt: OCI ref pulled+staged by the controller
+    image_cache_dir: str = ""  # controller-local oras pull cache ("" → default)
     # OpenStack-only (optional / unused for the libvirt backend)
     cloud: str = ""
     flavor_name: str = ""
@@ -153,11 +160,14 @@ class _Host(BaseModel):
         gpu_pci_addresses: list[str] = []
         max_slots: int | None = None
         image_name: str | None = None
+        image_ref: str | None = None
 
     class _Backend(BaseModel):
         name: str
         type: str = "openstack"
-        image_name: str
+        image_name: str = ""
+        image_ref: str = ""
+        image_cache_dir: str = ""
         min_ready: int = 1
         max_total: int = 2
         # OpenStack-only (optional for the libvirt backend)
@@ -244,6 +254,8 @@ def settings_customise_sources(
             type=s.backend.type,
             cloud=s.backend.cloud,
             image_name=s.backend.image_name,
+            image_ref=s.backend.image_ref,
+            image_cache_dir=s.backend.image_cache_dir,
             flavor_name=s.backend.flavor_name,
             network_name=s.backend.network_name,
             keypair=s.backend.keypair,
@@ -262,6 +274,7 @@ def settings_customise_sources(
                     gpu_pci_addresses=tuple(h.gpu_pci_addresses),
                     max_slots=h.max_slots,
                     image_name=h.image_name,
+                    image_ref=h.image_ref,
                 )
                 for h in s.backend.hosts
             ),

src/husk/controller.py

@@ -37,7 +37,10 @@
 # the backend/github/server objects and needs a restart — apply_reloaded_config
 # warns about and ignores changes to those rather than half-applying them.
 _HOT_RELOAD: dict[str, tuple[str, ...]] = {
-    "backend": ("min_ready", "max_total"),
+    # image_ref is hot: a new ref is picked up by sync_images on the next tick,
+    # which stages the new golden and drains slots onto it (see image-pipeline.md
+    # Phase C). Per-host image_ref overrides are still restart-only.
+    "backend": ("min_ready", "max_total", "image_ref"),
     "controller": ("shrink_ticks",),
     "timeouts": (
         "poll_interval_sec",
@@ -189,10 +192,31 @@ def apply_reloaded_config(self, new: Config) -> None:
             self.cfg = dataclasses.replace(self.cfg, **replacements)
             log.info("config reload applied: %s", "; ".join(changes))
 
+    def _sync_images(self) -> None:
+        """Hand the backend the current image config so it can stage the golden
+        image to every host (and GC orphans). Optional: backends with no image
+        delivery (OpenStack/fake) don't implement it. A failure here must not kill
+        the tick — we log and proceed with whatever image the hosts already hold."""
+        fn = getattr(self.backend, "sync_images", None)
+        if fn is None:
+            return
+        try:
+            fn(self.cfg.backend)
+        except Exception:
+            log.warning(
+                "image sync failed; continuing with current host image",
+                exc_info=True,
+            )
+
     # ----------------------------------------------------------------- tick
     def tick(self) -> ControllerState | None:
         now = self._clock()
 
+        # 0. IMAGE SYNC — ensure each host holds the configured golden image
+        #    before any create/rebuild this tick (a no-op once synced; on a ref
+        #    change it stages the new image and slots drain onto it below).
+        self._sync_images()
+
         # 1. FAIL-SAFE SNAPSHOT — a raise aborts the whole tick (no mutations).
         try:
             slots = self.backend.list_slots()
@@ -287,12 +311,20 @@ def tick(self) -> ControllerState | None:
                     log.warning("slot %s busy past max_job_duration; stopping", s.id)
                     self._safe(lambda: self.backend.stop_slot(s), f"stop {s.id}")
             elif state is SlotState.IDLE:
-                if (
-                    runner
-                    and self._state_age(s.id, now) > self.cfg.timeouts.idle_timeout_sec
-                ):
+                stale = s.image_stale
+                idle_timed_out = (
+                    self._state_age(s.id, now) > self.cfg.timeouts.idle_timeout_sec
+                )
+                if runner and (stale or idle_timed_out):
+                    # Deregister the (idle, no-job) runner so GitHub stops
+                    # dispatching to it; the slot then recycles — rebuilding onto
+                    # the current golden, which clears a stale image. Same drain
+                    # path as the idle-timeout reap, just also triggered by a
+                    # config image_ref change.
                     log.info(
-                        "slot %s idle past idle_timeout; deregistering runner", s.id
+                        "slot %s idle; deregistering runner (%s)",
+                        s.id,
+                        "stale image" if stale else "idle_timeout",
                     )
                     self._safe(
                         lambda: self.github.delete_runner(runner.id), "delete_runner"