From b83d305f4559943ade029a534f9ed09994d7e999 Mon Sep 17 00:00:00 2001 From: Willy Markuske Date: Thu, 4 Jun 2026 10:48:48 -0700 Subject: [PATCH] feat: optional /dev/fuse injection via CW_INJECT_FUSE_DEVICE Add an opt-in flag that injects /dev/fuse (with the device-cgroup rw rule) into device-plugin allocations, so rootless apptainer/squashfuse works in NON-privileged GPU pods (e.g. SUNK slurmd) without a privileged securityContext or a separate fuse device plugin. On cgroup-v2 + containerd 2.x, a plain hostPath mount of /dev/fuse is insufficient: open() is gated by the per-container device cgroup, which only the kubelet can populate at pod creation (via a device plugin, DRA, or privileged). This appends a /dev/fuse DeviceSpec to apiDeviceSpecs() when CW_INJECT_FUSE_DEVICE=true, so the kubelet adds the node + cgroup allow rule. - internal/plugin/server.go: env-gated DeviceSpec append (PassDeviceSpecs path) - chart: injectFuseDevice value renders CW_INJECT_FUSE_DEVICE on the container Default behavior unchanged (flag off). Covers the PassDeviceSpecs / volume-mounts device-list strategy; CDI strategies (updateResponseForCDI) are a follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../templates/daemonset-device-plugin.yml | 4 ++++ .../helm/nvidia-device-plugin/values.yaml | 7 +++++++ internal/plugin/server.go | 16 ++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/deployments/helm/nvidia-device-plugin/templates/daemonset-device-plugin.yml b/deployments/helm/nvidia-device-plugin/templates/daemonset-device-plugin.yml index eb9afabde..39558da5e 100644 --- a/deployments/helm/nvidia-device-plugin/templates/daemonset-device-plugin.yml +++ b/deployments/helm/nvidia-device-plugin/templates/daemonset-device-plugin.yml @@ -169,6 +169,10 @@ spec: - name: PASS_DEVICE_SPECS value: {{ .Values.compatWithCPUManager | quote }} {{- end }} + {{- if .Values.injectFuseDevice }} + - name: CW_INJECT_FUSE_DEVICE + value: "true" + {{- end }} {{- if typeIs "string" .Values.deviceListStrategy }} - name: DEVICE_LIST_STRATEGY value: {{ .Values.deviceListStrategy }} diff --git a/deployments/helm/nvidia-device-plugin/values.yaml b/deployments/helm/nvidia-device-plugin/values.yaml index f32f7c9e1..e33ee8d0c 100644 --- a/deployments/helm/nvidia-device-plugin/values.yaml +++ b/deployments/helm/nvidia-device-plugin/values.yaml @@ -42,6 +42,13 @@ dpDisableHealthchecks: null imexChannelIds: null imexRequired: null +# CoreWeave: when true, inject /dev/fuse (with the device-cgroup rw rule) into every +# allocation, so rootless apptainer/squashfuse works in NON-privileged GPU pods (e.g. +# SUNK slurmd) without a privileged securityContext or a separate fuse device plugin. +# Sets CW_INJECT_FUSE_DEVICE=true on the device-plugin container. Requires a non-CDI +# device-list strategy (PassDeviceSpecs / volume-mounts); CDI strategies not yet covered. +injectFuseDevice: false + nameOverride: "" fullnameOverride: "" namespaceOverride: "" diff --git a/internal/plugin/server.go b/internal/plugin/server.go index 6b55589b3..4afee440f 100644 --- a/internal/plugin/server.go +++ b/internal/plugin/server.go @@ -541,5 +541,21 @@ func (plugin *nvidiaDevicePlugin) apiDeviceSpecs(devRoot string, ids []string) [ specs = append(specs, spec) } + // CoreWeave: optionally inject /dev/fuse into every allocation so rootless + // apptainer/squashfuse works in NON-privileged GPU pods (e.g. SUNK slurmd). This + // adds the device-cgroup rw rule that a plain hostPath mount cannot provide on + // cgroup-v2 + containerd 2.x. Opt-in via CW_INJECT_FUSE_DEVICE=true; default + // behavior is unchanged. /dev/fuse lives at the host /dev root, not under devRoot. + // NOTE: this path runs only when PassDeviceSpecs is enabled (non-CDI device-list + // strategy). CDI strategies allocate via updateResponseForCDI() and are not yet + // covered — see the chart docs / follow-up for CDI support. + if os.Getenv("CW_INJECT_FUSE_DEVICE") == "true" { + specs = append(specs, &pluginapi.DeviceSpec{ + ContainerPath: "/dev/fuse", + HostPath: "/dev/fuse", + Permissions: "rw", + }) + } + return specs }