diff --git a/Cargo.lock b/Cargo.lock index a4c7430b9c..2e1d017d18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4220,14 +4220,17 @@ dependencies = [ "miette", "nix 0.29.0", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", + "openshell-sandbox-backend", "opentelemetry", "opentelemetry_sdk", "prost-types", "rustix 1.1.4", "serde", "serde_json", + "tar", "temp-env", "thiserror 2.0.18", "tokio", @@ -4238,6 +4241,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", + "uuid", ] [[package]] diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 0889dc3e60..a2bb25c312 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -5,6 +5,15 @@ gateway. A supported runtime provisions `openshell-sandbox` inside the workload, `openshell-supervisor` outside it, a protected channel between them, and an independent outer network fence. Drivers do not implement policy evaluation. +Podman provisions a paired workload and supervisor container using its native +libpod API. The workload uses `network=none`; the external supervisor alone joins +the configured network. A per-sandbox named volume carries their mutually +authenticated gRPC Unix socket, with supervisor credentials kept in its separate +filesystem. Both containers run as the resolved non-root identity with all +capabilities dropped. They share only a user namespace for volume ownership, +not PID, mount, or network namespaces. Podman owns paired lifecycle and health; +the common protocol owns process, identity, TCP, DNS, and forwarding semantics. + ## Driver Contract Each runtime receives a sandbox spec and canonical policy from the gateway and diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 8b3e014e8c..c49c309f65 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -17,6 +17,10 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } openshell-otel = { path = "../openshell-otel" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } +openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } +tar = "0.4" +uuid = { workspace = true } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 567abcbfcd..4ad9eedae6 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -1,463 +1,62 @@ -# Rootless Podman Networking +# Podman Networking -Deep-dive into how networking works in the Podman compute driver when running -rootless with pasta as the network backend. Covers the external tooling -(Podman, Netavark, pasta, aardvark-dns), the three nested namespace layers, and -the complete data paths for SSH, outbound traffic, and supervisor-to-gateway -communication. - -For the general Podman driver architecture, lifecycle, API surface, and driver -comparison, see [README.md](README.md). - -## Component Stack - -Podman's networking is composed of four independent projects: - -| Component | Language | Role | -|---|---|---| -| Podman | Go | Container runtime; orchestrates network lifecycle. | -| Netavark | Rust | Network backend; creates interfaces, bridges, firewall rules. | -| aardvark-dns | Rust | Authoritative DNS server for container name resolution. | -| pasta, part of passt | C | User-mode networking; L2-to-L4 socket translation for rootless containers. | - -The key split: rootful containers default to Netavark bridge networking with -real kernel interfaces, while rootless containers commonly use pasta user-mode -networking without needing host privileges. - -## How Netavark Works - -Netavark is invoked by Podman as an external binary. It reads a JSON network -configuration from STDIN and executes one of three commands: - -- `netavark setup ` creates interfaces, assigns IPs, and sets up - firewall rules for NAT and port-forwarding. -- `netavark teardown ` reverses setup and removes interfaces and - firewall rules. -- `netavark create` takes a partial network config and completes it by - assigning subnets and gateways. - -For rootful bridge networking: - -1. Podman creates a network namespace for the container. -2. Podman invokes `netavark setup` with the network config JSON. -3. Netavark creates a bridge, such as `podman0`, if it does not exist. The - default subnet is `10.88.0.0/16`. -4. Netavark creates a veth pair. One end goes into the container's netns and - the other attaches to the bridge. -5. Netavark assigns an IP from the subnet to the container's veth interface. -6. Netavark configures iptables or nftables rules for masquerade and port - mappings. -7. Netavark starts aardvark-dns when DNS is enabled, listening on the bridge - gateway address. - -```text -Host Kernel - | - +-- Bridge interface, such as "podman0" - | | - | +-- veth pair endpoint, host side, container 1 - | +-- veth pair endpoint, host side, container 2 - | - +-- Host physical interface, such as eth0 - | - +-- NAT, iptables or nftables rules managed by Netavark -``` - -Netavark also supports macvlan networks, where the container gets a -sub-interface of a physical host NIC with its own MAC address, and external -plugins via a documented JSON API. - -## How Pasta Works - -Unprivileged users cannot create network interfaces on the host. They cannot -create veth pairs, bridges, or iptables rules. Netavark's bridge approach -cannot work directly for rootless containers without an additional rootless -networking layer. - -Pasta, part of the `passt` project, operates in userspace and translates -between the container's L2 TAP interface and the host's L4 sockets. It requires -no capabilities or privileges. - -```text -Container Network Namespace - | - +-- TAP device, such as "eth0" - | ^ - | | L2 frames, Ethernet - | v - +-- pasta process, userspace - | - | Translation: L2 frames <-> L4 sockets - | - v - Host Network Stack, native TCP/UDP/ICMP sockets -``` - -For an outbound TCP connection from a container: - -1. The application calls `connect()` to an external address. -2. The kernel routes the packet through the default gateway to the TAP device. -3. Pasta reads the raw Ethernet frame from the TAP file descriptor. -4. Pasta parses L2/L3/L4 headers and identifies the TCP SYN. -5. Pasta opens a native TCP socket on the host and calls `connect()` to the - same destination. -6. When the host socket connects, pasta reflects the SYN-ACK back through the - TAP as an L2 frame. -7. For ongoing data transfer, pasta translates between TAP frames and the host - socket, coordinating TCP windows and acknowledgments between the two sides. - -Pasta does not maintain per-connection packet buffers. It reflects observed -sending windows and ACKs directly between peers. This is a thinner translation -layer than a full TCP/IP stack. - -### Built-in Services - -Pasta includes minimal network services so the container stack can -auto-configure: - -| Service | Purpose | -|---|---| -| ARP proxy | Resolves the gateway address to the host's MAC address. | -| DHCP server | Hands out a single IPv4 address, usually matching the host's upstream interface. | -| NDP proxy | Handles IPv6 neighbor discovery and SLAAC prefix advertisement. | -| DHCPv6 server | Hands out a single IPv6 address, usually matching the host's upstream interface. | - -By default there is no NAT. Pasta copies the host's IP addresses into the -container namespace. - -### Local Connection Bypass - -For connections between the container and the host, pasta implements a local -bypass path: - -- Packets with a local destination skip L2 translation. -- TCP uses `splice(2)`. -- UDP uses `recvmmsg(2)` and `sendmmsg(2)`. - -### Port Forwarding - -By default, pasta uses auto-detection. It scans `/proc/net/tcp` and -`/proc/net/tcp6` periodically and automatically forwards ports that are bound -and listening. Port forwarding is configurable through pasta options. - -### Security Properties - -Pasta is designed for rootless use: - -- No dynamic memory allocation after startup. -- All capabilities dropped, except `CAP_NET_BIND_SERVICE` when granted. -- Restrictive seccomp profile. -- Detaches into its own user, mount, IPC, UTS, and PID namespaces. -- No external dependencies beyond libc. - -### Inter-Container Limitation - -Unlike bridge networking, pasta containers are isolated from each other by -default. No virtual bridge connects them. Communication requires port mappings -through the host, pods with a shared network namespace, or opting into rootless -Netavark bridge networking with `podman network create`. - -## Three Nested Namespaces - -The Podman compute driver creates three layers of network isolation: +Only the external supervisor has external network connectivity. The workload +container uses `network=none`; its loopback DNS relay and TCP socket mediation +reach the supervisor through a protected Unix socket, not a veth or proxy +environment variable. ```text -Namespace 1: Host - | - pasta manages port forwarding, such as 127.0.0.1: - gateway listens on its configured bind address and port - | -Namespace 2: Rootless Podman network namespace, managed by pasta - | - Bridge "openshell", often 10.89.x.0/24 - aardvark-dns for container name resolution - | - Container netns - supervisor, proxy, and relay client run here - | -Namespace 3: Inner sandbox netns, created by supervisor - | - veth pair, such as 10.200.0.1 <-> 10.200.0.2 - nftables forces ordinary traffic through proxy - user workload runs here +workload container supervisor container +agent -> sandbox -- private UDS / gRPC -> policy proxy -> host network -> destination + | + +-- authenticated gateway callback ``` -Pasta bridges namespace 1 and 2. The veth pair bridges namespace 2 and 3. The -proxy at the boundary of namespace 2 and 3 enforces network policy. +## Outer network fence -### Layer 1 Pasta +The driver creates the workload without networks, host aliases, published +ports, or added capabilities. It checks the Podman inspect response before +launch and restart. The sandbox installs seccomp mediation and Landlock before +executing the agent. It does not create network namespaces, configure nftables, +or require `CAP_NET_ADMIN`. -At driver startup, the driver ensures a Podman bridge network exists: +TCP opens, TCP byte streams, DNS requests/replies, and lifecycle operations +share the authenticated gRPC channel. DNS is resolved and authorized by the +supervisor. General UDP is unsupported. -```rust -client.ensure_network(&config.network_name).await?; -``` +## Supervisor callback network -This creates a bridge network named `openshell` by default, with DNS enabled. -In rootless mode, this bridge can exist inside a user namespace managed by -pasta. The bridge IP range is not reliably routable from the host. - -```text -Host - | - 127.0.0.1:, pasta binds this on the host - | - pasta process, translates L4 sockets <-> L2 TAP frames - | - rootless network namespace - | - Bridge "openshell", such as 10.89.1.0/24 - | - +-- 10.89.1.1, bridge gateway and aardvark-dns - | - +-- veth to container netns - | - 10.89.1.2, container IP -``` - -### Layer 2 Container Networking - -The container spec configures: - -- `nsmode: "bridge"` to use the Podman bridge network. -- `networks` to attach to the configured bridge, `openshell` by default. -- `portmappings` with `host_port: 0`, `container_port: 2222`, and `protocol: - "tcp"` to publish the SSH compatibility port on an ephemeral host port. -- `hostadd` entries for `host.containers.internal` and - `host.openshell.internal`, using Podman's `host-gateway` resolver or the - configured `host_gateway_ip`. - -Pasta is not explicitly configured by the driver. The driver requests bridge -mode and logs the network backend that Podman reports at startup. - -The `host.containers.internal` hostname is injected into `/etc/hosts` so the -supervisor can reach the gateway on the host. Linux defaults to -`host-gateway`; macOS Podman machine defaults to `192.168.127.254`, gvproxy's -host-loopback IP, because older Podman machine images can fail to resolve -`host-gateway`. Override this with `host_gateway_ip` or -`OPENSHELL_PODMAN_HOST_GATEWAY_IP` when a Podman machine uses a non-standard -host-loopback address. - -If `OPENSHELL_GRPC_ENDPOINT` is empty, the driver auto-detects: - -```rust -if config.grpc_endpoint.is_empty() { - let scheme = if config.tls_enabled() { - "https" - } else { - "http" - }; - config.grpc_endpoint = - format!("{scheme}://host.containers.internal:{}", config.gateway_port); -} -``` - -The bridge gateway IP is not a stable substitute in rootless mode because it -can live inside the user namespace rather than on the host. - -Before the gateway binds its serving sockets, the driver reports the callback -listener required by the selected topology: - -- Rootful Linux Podman reports the configured bridge's gateway address exactly. -- Rootless Linux Podman explicitly reporting pasta requests the private IPv4 - source address selected by the host's default route. This avoids guessing - among private interfaces on a multihomed host. -- Rootless Linux Podman reporting slirp4netns, another named helper, or no - helper cannot use a direct local callback listener. The driver fails startup - unless `grpc_endpoint` names an explicitly remote endpoint. Supporting - slirp4netns requires a relay inside Podman's rootless network namespace. -- Podman Machine requests IPv4 loopback because gvproxy terminates the host - forwarding path there. -- An explicitly remote callback endpoint requests no additional local listener. - -On Linux, an explicit `host_gateway_ip` is reported exactly for rootful Podman -and rootless pasta because the driver maps both local callback aliases to that -literal. Other rootless helpers still fail closed. Podman Machine requests -gateway loopback because its configured address is guest-visible and gvproxy -terminates that route on host loopback. The gateway validates and binds every -accepted callback requirement. If the primary listener covers the requested -address, the gateway reuses it and relies on sandbox JWT authorization to limit -the supervisor's RPCs. Otherwise, it creates an additional listener that -exposes only the gateway's sandbox-callable gRPC methods. Operator, health, -reflection, and HTTP requests must use the primary listener. - -### Layer 3 Inner Sandbox Network Namespace - -Inside the container, the supervisor creates another network namespace for the -user workload: - -```text -Container on the Podman bridge - | - Supervisor process, running in container's default netns - | - +-- Proxy listener at the inner namespace gateway address - | - +-- veth pair - | - +-- Inner network namespace - | - sandbox-side veth address - | - default route -> supervisor-side veth address - | - user code runs here - | - nftables rules: - ACCEPT -> proxy TCP - ACCEPT -> loopback - ACCEPT -> established/related - LOG -> TCP SYN bypass attempts - REJECT -> TCP - LOG -> UDP bypass attempts - REJECT -> UDP -``` - -The supervisor uses `nsenter --net=` rather than `ip netns exec` to avoid sysfs -remount issues that arise under rootless Podman where real host -`CAP_SYS_ADMIN` is unavailable. - -For a policy with explicit `protocol: tcp` endpoints, this same inner namespace -also hosts policy DNS and transparent TCP capture. The supervisor answers only -policy-eligible names with epoch-scoped synthetic addresses, redirects TCP to -those synthetic ranges into its transparent listener, and leaves direct real-IP -dials subject to the terminal bypass fence. The Podman driver advertises this -substrate through its driver-owned runtime capability; sandbox image and policy -environment values cannot opt into it independently. - -The container spec preserves Podman's resolver search domains and options. -Policy DNS captures both UDP and TCP in the inner namespace, so it does not -depend on libc honoring `use-vc` and does not change ordinary short-name -resolution for sandboxes that do not use native TCP. - -A tmpfs is mounted at `/run/netns` in the container spec so the supervisor can -create named network namespaces. In rootless Podman this directory does not -exist on the host, so a private tmpfs gives the supervisor its own writable -`/run/netns` without needing host filesystem access. - -## Complete Data Paths - -### SSH Session - -```text -Client, openshell CLI - | - 1. gRPC: CreateSshSession -> gateway, returns token and connect_path - 2. HTTP CONNECT /connect/ssh to gateway - headers: x-sandbox-id, x-sandbox-token - | -Gateway - | - 3. Looks up SupervisorSession for sandbox_id - 4. Sends RelayOpen{channel_id} over ConnectSupervisor bidi stream - | - gRPC traverses host -> pasta translation -> container bridge - | -Supervisor inside container - | - 5. Receives RelayOpen, opens new RelayStream RPC back to gateway - 6. Sends RelayInit{channel_id} on the stream - 7. Connects to Unix socket /run/openshell/ssh.sock - 8. Bidirectional bridge: RelayStream <-> Unix socket - | -SSH daemon inside container, Unix socket only - | - 9. Authenticates. Access is gated by the relay chain. - 10. Spawns shell process - 11. Shell enters inner netns via setns(fd, CLONE_NEWNET) - | -User shell in sandbox netns -``` - -The SSH daemon listens on a Unix socket with restrictive permissions. The -published TCP port mapping exists in the container spec for compatibility and -health/debug paths. Normal SSH communication uses the gRPC reverse-connect relay -pattern. - -### Outbound HTTP Request - -```text -User code in inner netns - | - 1. curl https://api.example.com - HTTP_PROXY points at the local sandbox proxy - | - 2. TCP connect to proxy - allowed by nftables as the only ordinary egress destination - | - 3. HTTP CONNECT api.example.com:443 - | -Supervisor proxy in container netns - | - 4. Policy evaluation with process identity - 5. SSRF check - 6. Optional L7 TLS intercept and HTTP method/path inspection - | - 7. If allowed, TCP connect to api.example.com:443 - from the container netns - | - 8. Through Podman bridge -> pasta -> host -> internet -``` - -### Supervisor gRPC Callback - -The Podman driver auto-detects the callback endpoint scheme based on whether -TLS client certificates are configured. When the RPM's auto-generated PKI is in -place, the endpoint is `https://host.containers.internal:17670` and the -supervisor connects with mTLS. Without TLS configuration, it falls back to -`http://host.containers.internal:`. - -```text -Supervisor in container netns - | - 1. Connects to host.containers.internal: - with mTLS when OPENSHELL_TLS_* paths are set - | - 2. Routed through container default gateway - | - 3. Pasta translates L2 frame -> host L4 socket when rootless backend uses pasta - | - 4. Host TCP socket connects to gateway - | -Gateway - | - 5. TLS handshake when enabled - 6. ConnectSupervisor bidirectional stream established - 7. Heartbeats at the interval accepted by the gateway - 8. Reconnects with exponential backoff on failure - 9. Same gRPC channel reused for RelayStream calls -``` +The supervisor companion uses Podman's host network. Host-gateway aliases and +the upstream corporate proxy apply only to the supervisor. The gateway's SSH +tunnel uses the supervisor relay over its private Unix socket, so the driver +does not publish a supervisor port. -The gateway binds to `127.0.0.1:17670` by default in the RPM packaging. Client -certificates are auto-generated by `openshell-gateway generate-certs` on first -start and bind-mounted into sandbox containers by the Podman driver. +Rootful Podman uses the configured bridge and its gateway address. Rootless +local callbacks require the existing pasta path; slirp4netns or unknown helpers +require an explicitly remote `grpc_endpoint`. On macOS, Podman Machine provides +the runtime and host-loopback forwarding. -## Differences from the Kubernetes Driver +These runtime-managed network helpers are outside the workload trust boundary. +Sharing the workload's user namespace preserves volume UID/GID mapping; it +does not share the workload's PID, mount, or network namespaces. -| Aspect | Kubernetes | Podman, rootless pasta | -|---|---|---| -| Container or pod IP | Routable cluster-wide | Non-routable from the host in common rootless setups. | -| Network reachability | Pod IPs reachable from gateway | Bridge not reliably routable from host; requires host aliases or published ports. | -| Sandbox to gateway | Direct TCP to Kubernetes service or endpoint | `host.containers.internal` through bridge and rootless backend. | -| SSH transport | Reverse gRPC relay | Reverse gRPC relay. | -| Port publishing | Not needed for relay | Ephemeral host port remains in the container spec for compatibility and debug paths. | -| TLS | mTLS via Kubernetes secrets | mTLS via mounted client files, RPM defaults, or explicit configuration. | -| DNS | Kubernetes CoreDNS | Podman bridge DNS through aardvark-dns when DNS is enabled. | -| Network policy | Kubernetes network policy for pod ingress plus supervisor policy | nftables inside inner sandbox netns plus supervisor policy. | -| Supervisor delivery | Kubernetes driver managed pod image or template | OCI image volume mount. | -| Secrets | Kubernetes Secret volume and env vars | Per-sandbox JWT via Podman secret; TLS client materials from configured host files. | +## Troubleshooting -Both drivers use the same reverse gRPC relay for SSH transport. The most -important Podman-specific difference is network reachability: in rootless -Podman, the bridge network is not reliably routable from the host, so -host-to-container and container-to-host communication must use host aliases, -published ports, or the supervisor relay. +Inspect both containers with the same sandbox-ID label, distinguishing +`openshell.io/isolation-role=sandbox` from +`openshell.io/isolation-role=supervisor`. -## Port Assignments +- Sandbox fails its qualification probe: use its log to identify the denied + kernel/runtime primitive. Do not add capabilities or disable runtime seccomp. +- Sandbox cannot authenticate to supervisor: check the private channel volume, + matching user namespace mappings, and shared SELinux label. +- Supervisor cannot call back: inspect its configured gateway endpoint, + credentials, host network, and gateway callback listener. +- DNS or egress denied: inspect supervisor policy decisions. Do not add a + workload network, resolver bypass, or direct gateway route. +- Pair is not Ready: check the supervisor health socket and gateway session. + A running workload container alone does not establish readiness. -| Port | Component | Purpose | -|---|---|---| -| `17670` | Gateway | Default local gRPC and HTTP multiplexed server port. | -| `2222` | Sandbox | Container port mapping default for the SSH compatibility port. | -| `3128` | Sandbox proxy | HTTP CONNECT proxy inside the sandbox network model. | -| `0` | Host | Ephemeral host port requested for the container SSH compatibility port. | +See the [driver overview](README.md) and +[Podman runtime documentation](https://docs.podman.io/en/latest/markdown/podman-run.1.html) +for runtime options. diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 5781594158..912a79ac7a 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -1,489 +1,122 @@ # openshell-driver-podman -The Podman compute driver manages sandbox containers via the Podman REST API -over a Unix socket. It targets single-machine and developer environments where -rootless container isolation is preferred over a full Kubernetes cluster. The -driver runs in-process within the gateway server and delegates all sandbox -isolation enforcement to the `openshell-sandbox` supervisor binary, which is -sideloaded into each container via an OCI image volume mount. +The Podman compute driver runs inside the gateway and uses the native libpod +REST API over a Unix socket. Each sandbox has two independent containers: -When the gateway configures `[openshell.gateway.otlp]`, Podman compute-driver -spans export to the same OTLP/gRPC collector with the service name -`openshell-driver-podman`. The driver preserves the gateway trace context and -uses the same compute-driver RPC span names in its in-process and standalone -forms. +- `openshell-sandbox` owns the agent process in the workload container. +- `openshell-supervisor` evaluates policy, holds gateway credentials, and + proxies approved egress in a separate companion container. -`mise run gateway:podman` enables this export only when a local collector is -listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration -so the development gateway does not repeatedly report export failures. +The driver provisions placement, identity, credentials, transport, and lifecycle. +The shared isolation interface supplies exec, attach, signal, terminate, binary +identity, DNS, TCP, and loopback-forwarding semantics. -Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID and raw OCI `Config.User`. Container creation -uses that image ID with pulling disabled, preventing a mutable tag from changing -between inspection and launch. The supervisor runs as root, resolves omitted -policy identity fields from the image declaration, and drops only agent -children to the completed identity. Named OCI components remain names after -validation; a missing group is filled with the user's numeric primary GID. Explicit -`process.run_as_user` and `process.run_as_group` values take precedence -independently. +## Runtime posture -For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md). - -## Stop and Start - -Stop stops the managed container without deleting it. The per-sandbox named -workspace volume, token and proxy-auth secrets, labels, and container metadata -remain intact. Start starts the same container and reuses the same named -volume. Stopped managed containers remain visible through list and watch -reconciliation. Delete remains responsible for removing the container, -driver-owned secrets, and workspace volume. - -The stop call waits until Podman reports the container as stopped or exited. -This keeps an immediate start from racing a rootless Podman stop that is still -finishing after its API request returns. - -Graceful gateway shutdown sends `StopSandbox` for each sandbox whose persisted -phase requires running compute without changing that persisted intent. On -startup, the gateway sends an idempotent `StartSandbox` request for the same -sandboxes, restarting their retained containers. Explicitly stopped sandboxes -remain excluded. - -## Architecture - -The Podman driver communicates with the Podman daemon over a Unix socket and -delegates sandbox isolation to the supervisor binary running inside each -container. - -```mermaid -graph TB - CLI["openshell CLI"] -->|gRPC| GW["Gateway Server
(openshell-server)"] - GW -->|in-process| PD["PodmanComputeDriver"] - PD -->|HTTP/1.1
Unix socket| PA["Podman API"] - PA -->|OCI runtime
crun/runc| C["Sandbox Container"] - C -->|image volume
read-only| SV["Supervisor Binary
/opt/openshell/bin/openshell-sandbox"] - SV -->|creates| NS["Nested Network Namespace
veth pair + proxy"] - SV -->|enforces| LL["Landlock + seccomp"] - SV -->|gRPC callback| GW -``` - -## Isolation Model - -The Podman driver provides the same protection layers as the other compute -drivers. The driver itself does not implement isolation primitives directly. It -configures the container so that the `openshell-sandbox` supervisor can enforce -them at runtime. - -### Container Security Configuration - -The container spec in `container.rs` sets these security-critical fields: - -| Setting | Value | Rationale | +| Property | Workload | Supervisor | |---|---|---| -| `user` | `0:0` | The supervisor needs root inside the container for namespace creation, proxy setup, Landlock, seccomp, and filesystem preparation. | -| `cap_drop` | Selected unneeded defaults | Podman's default capability set is already restricted. The driver drops capabilities the supervisor does not need. | -| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP`, `KILL` | Grants supervisor-only capabilities required for namespace setup, process identity, bypass diagnostics, child bounding-set cleanup, and forwarding shutdown signals to a workload that runs as the sandbox user. Policy DNS binds an unprivileged supervisor port and does not require `NET_BIND_SERVICE`. | -| `no_new_privileges` | `true` | Prevents privilege escalation after exec. | -| `seccomp_profile_path` | `unconfined` | The supervisor installs its own policy-aware BPF filter. A container-level profile can block Landlock/seccomp syscalls during setup. | -| `mounts` | Private tmpfs at `/run/netns` | Lets the supervisor create named network namespaces in rootless Podman. | -| CDI GPU devices | Opaque `driver_config.cdi_devices` values when set, otherwise the requested count of NVIDIA CDI GPUs selected in round-robin order. Local `/dev/dxg` permits `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. | Exposes requested GPUs to GPU-enabled sandbox containers. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | - -The restricted agent child does not retain these supervisor privileges. - -## Driver Config Mounts - -The gateway forwards the `podman` block from `--driver-config-json` to this -driver. The driver accepts user-supplied `mounts` entries with these Podman -mount types: - -- `bind`: mounts an absolute host path when `[openshell.drivers.podman]` - has `enable_bind_mounts = true`. -- `volume`: mounts an existing Podman named volume. The driver validates that - the volume exists before provisioning and never creates or removes it. Podman - local-driver volumes created with bind options are treated as host bind - mounts and require `enable_bind_mounts = true`. -- `tmpfs`: mounts an in-memory filesystem with optional `options`, - `size_bytes`, and `mode`. -- `image`: mounts an OCI image through Podman's image-volume API. The driver - pulls the image during provisioning using the sandbox image pull policy. - -Host bind mounts are disabled by default because they expose gateway host paths -to sandbox requests. The driver still uses internal bind mounts for configured -TLS material; per-sandbox gateway JWTs are delivered through Podman secrets. - -Podman `bind` mounts accept `source`, `target`, optional `read_only`, and an -optional `selinux_label` of `shared` (applies `:z`) or `private` (applies -`:Z`) for SELinux-enforcing hosts. User-supplied bind and volume mounts are -read-only by default; set `read_only: false` to make them writable. Podman -image and volume mounts do not support `subpath` in OpenShell driver config. -Mount `source` and `target` values must not contain surrounding whitespace. -Mount targets must be absolute container paths and must not replace -the workspace root (`/sandbox`) or overlap OpenShell supervisor files, -`/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. - -Example named-volume usage: - -```shell -podman volume create openshell-work - -openshell sandbox create \ - --driver-config-json '{"podman":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work"}]}}' \ - -- claude -``` - -### Capability Breakdown - -| Capability | Purpose | -|---|---| -| `SYS_ADMIN` | seccomp filter installation, namespace creation, and Landlock setup. | -| `NET_ADMIN` | Network namespace veth setup, IP address assignment, routes, and nftables. | -| `SYS_PTRACE` | Reading `/proc//exe` and walking process ancestry for binary identity. | -| `SYSLOG` | Reading `/dev/kmsg` for bypass-detection diagnostics. | -| `DAC_READ_SEARCH` | Reading `/proc//fd/` across UIDs so the proxy can resolve the binary responsible for a connection. | -| `SETPCAP` | Clearing the restricted child process capability bounding set before exec. | - -The driver intentionally keeps Podman's default `SETUID`, `SETGID`, `CHOWN`, -and `FOWNER` capabilities because the supervisor needs them to drop privileges -and prepare writable sandbox directories. It also keeps `SETPCAP` until child -setup so `drop_privileges()` can clear the child capability bounding set before -exec. It drops unneeded defaults such as -`DAC_OVERRIDE`, `FSETID`, `KILL`, `NET_RAW`, `SETFCAP`, -and `SYS_CHROOT`. - -## Supervisor Sideloading - -The supervisor binary is delivered to sandbox containers via Podman's OCI image -volume mechanism, distinct from both the Kubernetes pod-volume approach and the -VM's embedded guest bundle. - -```mermaid -sequenceDiagram - participant D as PodmanComputeDriver - participant P as Podman API - participant C as Sandbox Container - - D->>P: pull_image(supervisor, "missing") - D->>P: create_container(spec with image_volumes) - Note over P: Podman resolves image_volumes at
libpod layer before OCI spec generation - P->>C: Mount supervisor image at /opt/openshell/bin (read-only) - D->>P: start_container - C->>C: entrypoint: /opt/openshell/bin/openshell-sandbox -``` - -The supervisor image from `deploy/docker/Dockerfile.supervisor` provides the -static `openshell-sandbox` binary at `/openshell-sandbox`. -Mounting that image at `/opt/openshell/bin` makes the binary available as -`/opt/openshell/bin/openshell-sandbox`. - -The container spec sets that binary as the entrypoint. This avoids relying on -the sandbox image entrypoint or command, which might otherwise append the -supervisor path as an argument to an image-provided shell. - -## TLS - -When all three Podman TLS paths are set, the driver treats sandbox callbacks as -mTLS callbacks: - -- `OPENSHELL_PODMAN_TLS_CA` -- `OPENSHELL_PODMAN_TLS_CERT` -- `OPENSHELL_PODMAN_TLS_KEY` - -The driver validates that the TLS paths are provided as a complete set. Partial -configuration fails early instead of silently falling back to plaintext. - -When enabled, the driver: - -1. Switches the auto-detected endpoint scheme from `http://` to `https://`. -2. Bind-mounts the client cert files read-only into the container at - `/etc/openshell/tls/client/`. -3. Sets `OPENSHELL_TLS_CA`, `OPENSHELL_TLS_CERT`, and `OPENSHELL_TLS_KEY` to - the container-side paths. - -The supervisor reads these env vars and uses them to establish an mTLS -connection back to the gateway. On SELinux systems, the bind mounts include -Podman's shared relabel option so the container process can read the files. - -The RPM packaging auto-generates a self-signed PKI on first start via -`openshell-gateway generate-certs`. Client certs are placed in the CLI -auto-discovery directory (`~/.config/openshell/gateways/openshell/mtls/`) so -the CLI connects with mTLS without manual configuration. See -`deploy/rpm/CONFIGURATION.md` for the full RPM configuration reference. - -## Network Model - -Sandbox network isolation uses a two-layer approach: a Podman bridge network -for container-to-host communication, and a nested network namespace created by -the supervisor for sandbox process isolation. - -```mermaid -graph TB - subgraph Host - GW["Gateway Server
127.0.0.1:17670"] - PS["Podman Socket"] - end - - subgraph Bridge["Podman Bridge Network (10.89.x.x)"] - subgraph Container["Sandbox Container"] - SV["Supervisor
(root in user ns)"] - subgraph NestedNS["Nested Network Namespace"] - SP["Sandbox Process
(resolved non-root identity)"] - VE2["veth1: 10.200.0.2"] - end - VE1["veth0: 10.200.0.1
(CONNECT proxy)"] - SV --- VE1 - VE1 ---|veth pair| VE2 - end - end - - GW -.->|SSH via supervisor relay
gRPC session| SV - SV -->|gRPC callback via
host.containers.internal| GW - SP -->|all egress via proxy| VE1 +| UID/GID | Pinned non-root workload identity | Same mapped identity | +| Capabilities | Drop all; add none | Drop all; add none | +| Seccomp | Runtime default plus sandbox-installed filters | Runtime default | +| Network | `none`; loopback only | Podman host network | +| Gateway JWT and upstream credentials | Never mounted | Podman secrets | +| User volumes and CDI devices | Workload only | Never mounted | +| Channel | Private named volume, writable | Same volume, read-only | + +Podman creates the namespaces and volume ownership before the workload runs. +Rootless operation uses the operator's Podman service and subordinate-ID +configuration; it does not require adding capabilities to either container. +The supervisor joins the workload's **user namespace only** to preserve UID/GID +mapping for shared-volume access. PID, mount, and network namespaces remain +separate. The channel volume uses shared SELinux relabeling (`:z`). + +Before starting either container, the driver uploads volume-relative archives +directly to the channel and workspace volume destinations. A rootfs upload on a +stopped Podman container does not populate nested named volumes. Restart restores +only the channel bootstrap into the existing channel volume, preserving the +workspace. The workload starts before the supervisor so its user namespace exists +when the supervisor joins it; a stopped supervisor resolves that namespace again +on its next start. + +The runtime must pass the sandbox's unprivileged enforcement probe, including +nested seccomp notification and Landlock. Unsupported runtime defaults fail +closed; do not switch to an unconfined profile or add capabilities. + +## Protected channel and network enforcement + +```text +agent -> openshell-sandbox === authenticated gRPC / private UDS === supervisor -> network + network=none TCP, DNS, control streams policy + gateway JWT ``` -Key points: - -- Bridge network: created by `client.ensure_network()` with DNS enabled. - Containers on the bridge can see each other at L3, but sandbox processes - cannot because they are isolated inside the nested netns. -- Nested netns: the supervisor creates a private `NetworkNamespace` with a veth - pair. Sandbox processes enter this netns via `setns(fd, CLONE_NEWNET)` in the - `pre_exec` hook, forcing ordinary traffic through the CONNECT proxy. -- Policy DNS and transparent TCP: the driver advertises the complete - `policy-dns-transparent-tcp` substrate. For explicit `protocol: tcp` - endpoints, the supervisor installs namespace-local DNS listeners, synthetic - routes, and TCP redirect rules before starting the workload. The container - disables Podman's implicit DNS search suffix so policy DNS evaluates the - exact endpoint name requested by the workload, and asks libc to use the - policy DNS TCP listener to avoid rootless Podman's nested UDP NAT return - path. -- Port publishing: the container spec still requests `host_port: 0` for the - configured SSH port. The gateway SSH tunnel uses the supervisor relay rather - than connecting directly to the published port. -- Host gateway: `host.containers.internal` and `host.openshell.internal` are - injected into `/etc/hosts` so containers can reach services on the gateway - host. Linux defaults to Podman's `host-gateway` resolver. macOS Podman - machine defaults to gvproxy's host-loopback IP, `192.168.127.254`, because - stale Podman machines may fail to resolve `host-gateway`. -- nsenter: the supervisor uses `nsenter --net=` instead of `ip netns exec` for - namespace operations, avoiding the sysfs remount path that fails in rootless - containers. - -See [NETWORKING.md](NETWORKING.md) for the rootless Podman networking deep dive. - -## Supervisor Relay - -Podman follows the same end-to-end contract as the Kubernetes and VM drivers -for the in-container SSH relay: gateway config to `PodmanComputeConfig` to -sandbox environment to supervisor session registration on that path. - -1. `[openshell.drivers.podman].ssh_socket_path` is deserialized into - `PodmanComputeConfig::ssh_socket_path` when the gateway builds the in-process - driver. The field defaults to `/run/openshell/ssh.sock` when omitted. -2. `build_env()` in `container.rs` sets `OPENSHELL_SSH_SOCKET_PATH` to that - value, alongside required vars such as `OPENSHELL_ENDPOINT` and - `OPENSHELL_SANDBOX_ID`. These driver-controlled entries overwrite template - environment variables to prevent spoofing. -3. The supervisor reads `OPENSHELL_SSH_SOCKET_PATH` and uses it for the Unix - socket the gateway's SSH stack bridges to. - -The standalone `openshell-driver-podman` binary sets the same struct field from -`OPENSHELL_SANDBOX_SSH_SOCKET_PATH`. - -## Credential Injection - -Sandboxes authenticate to the gateway via mTLS using client materials bind- -mounted into the container from a Podman secret. No shared per-request secret -is injected as an environment variable. - -| Credential | Mechanism | Visible in `inspect`? | Visible in `/proc//environ`? | -|---|---|---|---| -| mTLS client cert/key | Bind-mounted file paths (`OPENSHELL_TLS_*` env vars point at them) | Yes (paths only) | Yes (paths only) | -| Sandbox identity | Plaintext env var | Yes | Yes | -| gRPC endpoint | Plaintext env var, override-protected | Yes | Yes | -| Supervisor relay socket path | Plaintext env var, override-protected | Yes | Yes | - -The `build_env()` function inserts user-supplied variables first, then -unconditionally overwrites all security-critical variables to prevent spoofing -via sandbox templates: - -- `OPENSHELL_SANDBOX` -- `OPENSHELL_SANDBOX_ID` -- `OPENSHELL_ENDPOINT` -- `OPENSHELL_SSH_SOCKET_PATH` -- `OPENSHELL_CONTAINER_IMAGE` -- `OPENSHELL_MAIN_PROCESS_SPEC` - -## Sandbox Lifecycle - -### Creation Flow - -```mermaid -sequenceDiagram - participant GW as Gateway - participant D as PodmanComputeDriver - participant P as Podman API - - GW->>D: create_sandbox(DriverSandbox) - D->>D: validate name + id - D->>D: validated_container_name() - - D->>P: pull_image(supervisor, "missing") - D->>P: pull_image(sandbox_image, policy) - - D->>P: create_volume(workspace) - Note over D: On failure below, rollback volume - - D->>P: create_container(spec) - alt Conflict (409) - D->>P: remove_volume - D-->>GW: AlreadyExists - end - Note over D: On failure below, rollback container + volume - - D->>P: start_container - D-->>GW: Ok -``` - -Each step rolls back previously-created resources on failure. The Conflict path -cleans up the volume because it is keyed by the new sandbox's ID, not the -conflicting container's ID. - -### Readiness and Health - -The container `healthconfig` marks the sandbox healthy when any of these -signals succeeds: - -- Legacy marker file `/var/run/openshell-ssh-ready`. -- `test -S` on the configured supervisor Unix socket path. -- The prior TCP check for a listener on the in-container SSH port. - -The Unix socket check allows relay-only backend readiness when the supervisor -exposes the socket without the old marker or published-port signal. Omitting -`health_check_interval_secs` disables these Podman/conmon probes, but it does -not bypass public readiness gating: the gateway keeps a backend-ready sandbox -in `Provisioning` with `SupervisorNotConnected` until its supervisor control -session is connected. - -### Deletion Flow - -1. Validate `sandbox_name` and stable `sandbox_id` from `DeleteSandboxRequest`. -2. Best-effort inspect cross-checks the container label when present, but - cleanup remains keyed by the request `sandbox_id`. -3. Best-effort stop, ignoring the stop result. -4. Force-remove the container. -5. Remove workspace volume derived from the request `sandbox_id`, warning on - failure and continuing. - -If the container is already gone during inspect or remove, the driver still -performs idempotent volume cleanup using the request `sandbox_id` and -returns `Ok(false)` for the container-delete result. This prevents leaked -Podman resources after out-of-band container removal or label drift. - -## Configuration - -| Environment Variable | CLI Flag | Default | Description | -|---|---|---|---| -| `OPENSHELL_PODMAN_SOCKET` | `--podman-socket` | Probes known local Podman API sockets and uses the first responsive socket, then falls back to asking the `podman` CLI for the host-side socket. Fails to start if neither finds one. | Podman API Unix socket path. | -| `OPENSHELL_SANDBOX_IMAGE` | `--sandbox-image` | From gateway config | Default OCI image for sandboxes. | -| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `if_not_present` | Pull policy: `always`, `if_not_present`, `never`, or `newer`. | -| `OPENSHELL_GRPC_ENDPOINT` | `--grpc-endpoint` | Auto-detected via `host.containers.internal` | Gateway gRPC endpoint for sandbox callbacks. | -| `OPENSHELL_GATEWAY_PORT` | `--gateway-port` | `17670` | Gateway port used for endpoint auto-detection by the standalone binary. | -| `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. | -| `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | -| `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | -| `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Omission uses OpenShell's `2048` default; explicit `0` is invalid. | -| `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | -| `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | -| `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | -| `OPENSHELL_PODMAN_TLS_KEY` | `--podman-tls-key` | unset | Host path to the client private key mounted for sandbox mTLS. | -| `OPENSHELL_SANDBOX_HTTPS_PROXY` | `--sandbox-https-proxy` | unset | Corporate forward proxy URL for the supervisor's upstream TLS dials, chained with HTTP CONNECT. Credential-free `http://host:port` and `https://host:port` URLs are supported (scheme and port required). For an `https://` proxy the supervisor TLS-wraps the proxy connection, verifying the proxy certificate against the built-in and system roots plus `--sandbox-proxy-ca-bundle`. Plain-HTTP requests always dial directly. | -| `OPENSHELL_SANDBOX_NO_PROXY` | `--sandbox-no-proxy` | unset | Comma-separated `NO_PROXY` list (hostnames, domain suffixes, IPs, CIDRs, each with an optional `:port` qualifier) dialed directly instead of through the corporate proxy. IP/CIDR entries also match hostnames through their validated DNS resolution. | -| `OPENSHELL_SANDBOX_PROXY_AUTH_FILE` | `--sandbox-proxy-auth-file` | unset | Path to a file containing the proxy credentials as `user:pass`. Staged as a root-only Podman secret so credentials never appear in config or container metadata. Requires the insecure-auth acknowledgement below. | -| `OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE` | `--sandbox-proxy-auth-allow-insecure` | unset | Explicit acknowledgement (`true`) that the credential is sent as cleartext Basic auth over the plain-TCP connection to the `http://` proxy. Required when the auth file is set with an `http://` proxy; not required for `https://` proxies (the credential travels inside the verified TLS session) but tolerated if set. Rejected when no auth file is configured. | -| `OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME` | `--sandbox-proxy-connect-by-hostname` | unset | Send the destination hostname in CONNECT requests instead of a validated IP. Last resort for proxies whose ACLs filter on hostnames: the proxy then resolves the name itself, so sandbox SSRF/`allowed_ips` validation no longer binds the connection. | -| `OPENSHELL_PODMAN_USERNS` | `--userns` | unset | User namespace mode for sandbox containers (e.g. `auto`). When unset, containers use the default user namespace. | -| `OPENSHELL_SANDBOX_PROXY_CA_BUNDLE` | `--sandbox-proxy-ca-bundle` | unset | Path (on the gateway host) to a PEM CA bundle trusted for the corporate proxy. Bind-mounted read-only into the sandbox (a CA certificate is not secret). Trusted for the `https://` proxy TLS handshake and, because TLS-intercepting proxies re-sign tunneled certificates, folded into the sandbox trust bundle and upstream verification. Requires a proxy URL; the file must exist and hold at least one certificate. | - -Through the gateway, the same settings are the `https_proxy`, `no_proxy`, -`proxy_auth_file`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, -and `proxy_ca_bundle` keys under `[openshell.drivers.podman]`; see -`docs/reference/gateway-config.mdx`. - -`provider_spiffe_workload_api_socket` accepts either an absolute host UNIX -Workload API socket, projected through a dedicated read-only mount, or an -explicit container-reachable `tcp:IP:port` endpoint. The driver sets the -supervisor's `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` accordingly. -`app_armor_profile` shares the canonical -`RuntimeDefault`, `Unconfined`, or `Localhost/` model with Docker and -Kubernetes. When omitted, the driver sends no override and preserves Podman's -runtime-selected profile. Set `Unconfined` explicitly only when the deployment -requires the supervisor's mount setup to bypass that profile. Explicit confined -choices fail early when Podman reports AppArmor unavailable. - -This is an operator-owned egress boundary: the driver passes the settings on -the supervisor's command line, so sandbox and template environment — and any -`ENV` baked into the sandbox image — cannot override them, and the -conventional `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` variables a sandbox -controls do not steer it. Credentials must be supplied through -`proxy_auth_file`; an inline `user:pass@` in the URL is rejected at startup. - -Basic auth over an `http://` proxy is cleartext on the wire: anyone on the -network path between the sandbox host and the proxy can recover the -credential. Setting `proxy_auth_file` therefore requires -`proxy_auth_allow_insecure = true`; both the driver and the in-container -supervisor reject credentials without that explicit acknowledgement. - -CONNECT requests target a validated resolved IP by default, so the proxy -performs no DNS resolution and the tunnel stays bound to the address that -passed the sandbox's SSRF and `allowed_ips` checks; the hostname still -travels inside the tunnel (TLS SNI, application `Host`). In split-horizon -networks, point the gateway host at the corporate resolver. Set -`proxy_connect_by_hostname = true` only when the proxy's ACLs filter on -hostnames and reject IP CONNECT targets — it re-opens proxy-side DNS -resolution, making the proxy's ACLs the effective egress control. - -## Rootless-Specific Adaptations - -The Podman driver is designed for rootless operation. The following adaptations -matter compared to cluster or rootful runtimes: - -1. subuid/subgid preflight check: on non-macOS hosts, `check_subuid_range()` in - `driver.rs` warns operators if `/etc/subuid` or `/etc/subgid` entries are - missing for the current user. This is not a hard error because some systems - use LDAP or other mechanisms. macOS skips the check because `podman machine` - runs the Podman service inside a Linux VM. -2. cgroups v2 requirement: the driver refuses to start if cgroups v1 is - detected. Rootless Podman requires the unified cgroup hierarchy. -3. `nsenter` for namespace operations: `openshell-sandbox` uses - `nsenter --net=` instead of `ip netns exec` to avoid the sysfs remount path - that requires real `CAP_SYS_ADMIN` in the host user namespace. -4. `DAC_READ_SEARCH` capability: required for the proxy to read - `/proc//fd/` across UIDs within the user namespace. -5. `SETUID` and `SETGID` capabilities: kept from Podman's default capability - set so `drop_privileges()` can call `setuid()` and `setgid()`. -6. `host.containers.internal`: used instead of Docker's `host.docker.internal` - for container-to-host communication. The driver also injects the - OpenShell-owned `host.openshell.internal` alias. -7. Ephemeral port publishing: the SSH compatibility port uses `host_port: 0` - because the bridge network IP is not reliably routable from the host in - rootless mode. -8. tmpfs at `/run/netns`: a private tmpfs lets the supervisor create named - network namespaces via `ip netns add`. - -## Implementation References - -- Gateway integration: `crates/openshell-gateway/src/lib.rs` registers the - driver factory and constructs `PodmanComputeConfig` from the generic server - build context. -- Server configuration: `crates/openshell-server/src/lib.rs` exposes the - backend-agnostic registry and factory context. -- Gateway relay path: `openshell-core` `Config::sandbox_ssh_socket_path` in - `crates/openshell-core/src/config.rs`. -- SSRF mitigation: `crates/openshell-core/src/net.rs`, - `crates/openshell-sandbox/src/proxy.rs`, and - `crates/openshell-server/src/grpc/policy.rs`. -- Sandbox supervisor: `crates/openshell-sandbox/src/` for Landlock, seccomp, - netns, proxy, and relay behavior shared by all drivers. -- Container engine abstraction: `tasks/scripts/container-engine.sh` for - build/deploy support across Docker and Podman. -- Supervisor image build: `deploy/docker/Dockerfile.supervisor`. +The workload has no external interface or published port. Seccomp socket +mediation carries TCP and DNS through one authenticated gRPC connection. +DNS remains supervisor-mediated; general UDP is unsupported. The driver sets +`net.ipv4.ip_unprivileged_port_start=0` in the isolated workload network +namespace so the sandbox's loopback DNS relay can bind port 53 without a +capability. No nftables or nested network namespace setup runs in the sandbox. + +The channel contains the sandbox bootstrap and sandbox-side TLS identity only. +Supervisor private keys and the runtime descriptor stay in the supervisor's private filesystem. +Landlock denies agent access to the top-level `/.openshell` control hierarchy. +The driver verifies Podman's reported `network=none` fence before launch and +restart. `host.containers.internal` and callback networking apply to the +supervisor, not the agent. + +Gateway callbacks use the existing sandbox JWT and optional configured mTLS +bundle. The sandbox/supervisor channel always uses its separate, per-sandbox +mutual TLS material. These are distinct authentication relationships. + +## Identity and trusted binaries + +Both workload and supervisor images are pinned by immutable image ID. The +driver reads account files from a stopped workload-image container; it never +executes the image to resolve an account. Policy identity fields override OCI +`USER` independently. Named users/groups resolve against that image, including +supplementary groups. Root and unresolved identities fail before provisioning. +Images must not prepopulate the reserved `/.openshell` hierarchy; this prevents +image-controlled symlinks from aliasing private control state into user mounts. + +`sandbox_runtime_image` supplies the statically linked musl +`/openshell-sandbox` binary. Podman's read-only image volume delivers it to the +workload; user-namespace modes that cannot use image volumes retain the trusted +binary extraction path. `supervisor_image` supplies the dynamically linked +glibc `/openshell-supervisor` binary outside the workload. Image and request +environment belong to agent children, never the supervisor process. + +## Lifecycle and readiness + +Create builds both stopped containers and stages the private archives before +starting either container. The sandbox does not execute the agent until the +supervisor authenticates and confirms the common boundary contract. Failed +creation removes only containers created by that attempt, then cleans up +driver-owned volumes and secrets. + +Stop retains both containers, workspace, channel, and secrets. Start restores +the consumed sandbox bootstrap from a copy in the supervisor's private +filesystem, verifies the fence, and starts the same pair. A failed supervisor +start stops the workload. Delete removes the companion first, then the workload, +channel, workspace, and driver-owned secrets. User-owned volumes are retained. + +Only workload containers appear in sandbox list/watch results. Readiness uses +the supervisor's private health socket; there is no shell, legacy marker, or +TCP-listener shortcut. Watch reconciliation and supervisor exit/removal events +stop a running workload whose companion is unavailable. The gateway also +requires the authenticated supervisor session before publishing Ready. + +## Mounts, GPUs, and configuration + +User `bind`, `volume`, `tmpfs`, and `image` mounts and CDI GPU selection remain +native Podman features and apply only to the workload. Bind mounts require the +operator's `enable_bind_mounts` opt-in. Reserved control paths and the workspace +root cannot be replaced. User-owned volumes are never created or deleted. + +See [gateway configuration](../../docs/reference/gateway-config.mdx) for +operator settings and [NETWORKING.md](NETWORKING.md) for callback networking. +The supervisor uses Podman's host network and owns the upstream proxy settings. +Omit `health_check_interval_secs` to disable Podman's periodic health command. +Explicit zero is invalid. OpenShell still gates readiness on the supervisor's +authenticated health signal. + +Gateway OTLP configuration continues to export compute-driver spans under the +`openshell-driver-podman` service, preserving gateway trace context. diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 0670a026dd..be5dcea0b9 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -177,6 +177,8 @@ pub struct ImageInspect { pub struct ImageConfig { #[serde(default)] pub user: String, + #[serde(default)] + pub env: Vec, } /// A container summary returned by the list API. @@ -450,6 +452,93 @@ impl PodmanClient { .await } + pub(crate) async fn create_typed_container( + &self, + spec: &(impl serde::Serialize + Sync), + ) -> Result { + #[derive(serde::Deserialize)] + struct Created { + #[serde(rename = "Id", alias = "ID")] + id: String, + } + let body = + serde_json::to_vec(spec).map_err(|error| PodmanApiError::Json(error.to_string()))?; + let (status, bytes) = self + .request_raw( + hyper::Method::POST, + "/libpod/containers/create", + "application/json", + body.into(), + ) + .await?; + if !status.is_success() { + return Err(error_from_response(status.as_u16(), &bytes)); + } + let created: Created = serde_json::from_slice(&bytes) + .map_err(|error| PodmanApiError::Json(error.to_string()))?; + validate_name(&created.id)?; + Ok(created.id) + } + + pub(crate) async fn copy_to_container( + &self, + name: &str, + destination: &str, + archive: Vec, + ) -> Result<(), PodmanApiError> { + validate_name(name)?; + let (status, bytes) = self + .request_raw( + hyper::Method::PUT, + &format!( + "/libpod/containers/{name}/archive?path={}", + url_encode(destination) + ), + "application/x-tar", + archive.into(), + ) + .await?; + if status.is_success() { + Ok(()) + } else { + Err(error_from_response(status.as_u16(), &bytes)) + } + } + + pub(crate) async fn verify_isolation_fence(&self, id: &str) -> Result<(), PodmanApiError> { + #[derive(serde::Deserialize)] + #[serde(rename_all = "PascalCase")] + struct HostConfig { + network_mode: String, + privileged: bool, + } + #[derive(serde::Deserialize)] + #[serde(rename_all = "PascalCase")] + struct FenceInspect { + host_config: HostConfig, + network_settings: NetworkSettings, + } + validate_name(id)?; + let inspected: FenceInspect = self + .request_json( + hyper::Method::GET, + &format!("/libpod/containers/{id}/json"), + None, + ) + .await?; + if inspected.host_config.network_mode != "none" + || inspected.host_config.privileged + || inspected + .network_settings + .networks + .keys() + .any(|name| name != "none") + { + return Err(PodmanApiError::InvalidInput("sandbox requires an unprivileged container with network mode none and no attached networks".into())); + } + Ok(()) + } + /// Start a container by name or ID. pub async fn start_container(&self, name: &str) -> Result<(), PodmanApiError> { validate_name(name)?; diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 9f139cde94..d0ca11669e 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -60,9 +60,11 @@ pub struct PodmanComputeConfig { pub host_gateway_ip: String, /// Container stop timeout in seconds (SIGTERM → SIGKILL). pub stop_timeout_secs: u32, - /// OCI image containing the openshell-sandbox supervisor binary. - /// Mounted read-only into sandbox containers at /opt/openshell/bin - /// using Podman's `type=image` mount. + /// OCI image containing the statically linked `openshell-sandbox` binary. + /// The driver extracts the binary from this image into a verified host + /// cache and mounts it read-only into each sandbox container. + pub sandbox_runtime_image: String, + /// OCI image containing the dynamically linked `openshell-supervisor` binary. pub supervisor_image: String, /// Host path to the CA certificate for sandbox mTLS. /// @@ -91,11 +93,11 @@ pub struct PodmanComputeConfig { /// Host path to a SPIFFE Workload API Unix socket exposed to sandbox /// supervisors for provider token exchange client assertions. pub provider_spiffe_workload_api_socket: Option, - /// `AppArmor` confinement requested for sandbox containers. Omission sends - /// no override and preserves Podman's runtime-selected profile. + /// `AppArmor` confinement requested for the workload container. Omission + /// sends no override and preserves Podman's runtime-selected profile. #[serde(default, skip_serializing_if = "Option::is_none")] pub app_armor_profile: Option, - /// Health check interval in seconds for sandbox containers. + /// Health check interval in seconds for supervisor containers. /// /// Podman runs the health check command at this interval to determine /// container readiness. Lower values detect readiness faster but @@ -469,6 +471,7 @@ impl Default for PodmanComputeConfig { network_name: DEFAULT_NETWORK_NAME.to_string(), host_gateway_ip: Self::default_host_gateway_ip(), stop_timeout_secs: DEFAULT_PODMAN_STOP_TIMEOUT_SECS, + sandbox_runtime_image: openshell_core::config::default_sandbox_runtime_image(), supervisor_image: openshell_core::config::default_supervisor_image(), guest_tls_ca: None, guest_tls_cert: None, @@ -503,6 +506,7 @@ impl std::fmt::Debug for PodmanComputeConfig { .field("network_name", &self.network_name) .field("host_gateway_ip", &self.host_gateway_ip) .field("stop_timeout_secs", &self.stop_timeout_secs) + .field("sandbox_runtime_image", &self.sandbox_runtime_image) .field("supervisor_image", &self.supervisor_image) .field("guest_tls_ca", &self.guest_tls_ca) .field("guest_tls_cert", &self.guest_tls_cert) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 541cb2f756..1ff3ab22b3 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -197,7 +197,7 @@ pub fn short_id(id: &str) -> String { // --------------------------------------------------------------------------- #[derive(Serialize)] -struct ContainerSpec { +pub struct ContainerSpec { name: String, image: String, labels: BTreeMap, @@ -212,18 +212,21 @@ struct ContainerSpec { entrypoint: Vec, command: Vec, user: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + groups: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + unsetenv: Vec, cap_drop: Vec, cap_add: Vec, no_new_privileges: bool, - seccomp_profile_path: String, - /// Podman's container create API accepts `AppArmor` through the dedicated - /// `apparmor_profile` `SpecGenerator` field. This is not Docker's - /// `security_opt` representation. #[serde(skip_serializing_if = "Option::is_none")] apparmor_profile: Option, + #[serde(skip_serializing_if = "String::is_empty")] + seccomp_profile_path: String, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + sysctl: BTreeMap, image_pull_policy: String, - #[serde(skip_serializing_if = "Option::is_none")] - healthconfig: Option, + healthconfig: HealthConfig, resource_limits: ResourceLimits, /// Env-type secrets: map of `ENV_VAR_NAME → secret_name`. /// Podman's libpod `SpecGenerator` uses `secret_env` (a flat map) for @@ -340,8 +343,6 @@ struct SecretMount { struct ResourceLimits { cpu: CpuLimits, memory: MemoryLimits, - // Podman's libpod API consumes the OCI LinuxResources shape. A Docker-style - // scalar PidsLimit is silently ignored and leaves the runtime default. #[serde(skip_serializing_if = "Option::is_none")] pids: Option, } @@ -486,7 +487,7 @@ fn build_env( config: &PodmanComputeConfig, image: &str, oci_user: &str, -) -> BTreeMap { +) -> Result, ComputeDriverError> { let spec = sandbox.spec.as_ref(); let template = spec.and_then(|s| s.template.as_ref()); @@ -511,10 +512,11 @@ fn build_env( user_env.insert(k.clone(), v.clone()); } } - env.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { + // User environment belongs exclusively to mediated workload children. In + // particular, never activate loader or policy overrides in the supervisor. + if !user_env.is_empty() { + let json = serde_json::to_string(&user_env) + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; env.insert(openshell_core::sandbox_env::USER_ENVIRONMENT.into(), json); } @@ -543,7 +545,7 @@ fn build_env( ); env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); let main_process = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(spec) - .expect("main process config serialization cannot fail"); + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; env.insert( openshell_core::sandbox_env::MAIN_PROCESS_SPEC.into(), main_process, @@ -615,7 +617,7 @@ fn build_env( ); } - env + Ok(env) } /// Merge labels from the sandbox template with required managed labels. @@ -665,12 +667,14 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) period: DEFAULT_CPU_PERIOD, }, memory: MemoryLimits { limit: mem_bytes }, - pids: config - .sandbox_pids_limit - .map(|limit| PidsLimits { limit: limit.get() }), + pids: podman_pids_limit(config.sandbox_pids_limit).map(|limit| PidsLimits { limit }), } } +fn podman_pids_limit(value: Option) -> Option { + value.map(std::num::NonZeroI64::get) +} + pub fn podman_driver_volume_mount_sources( sandbox: &DriverSandbox, enable_bind_mounts: bool, @@ -880,6 +884,7 @@ fn validate_podman_driver_mounts( } }; driver_mounts::validate_container_mount_target(target)?; + driver_mounts::validate_mount_control_path(target, "/.openshell")?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( @@ -951,14 +956,6 @@ fn validate_tmpfs_options(options: &[String]) -> Result, String> { .collect() } -fn podman_apparmor_profile(profile: Option<&openshell_core::AppArmorProfile>) -> Option { - match profile { - None | Some(openshell_core::AppArmorProfile::RuntimeDefault) => None, - Some(openshell_core::AppArmorProfile::Unconfined) => Some("unconfined".to_string()), - Some(openshell_core::AppArmorProfile::Localhost(profile)) => Some(profile.clone()), - } -} - /// Build the Podman container creation JSON spec. #[cfg(test)] #[must_use] @@ -1025,6 +1022,7 @@ pub fn build_container_spec_with_token_and_gpu_devices( } #[allow(clippy::too_many_arguments)] +#[cfg(test)] pub fn build_container_spec_for_image( sandbox: &DriverSandbox, config: &PodmanComputeConfig, @@ -1036,10 +1034,36 @@ pub fn build_container_spec_for_image( supervisor_bin_path: Option<&Path>, tls_secret_names: Option<&[String; 3]>, ) -> Result { + serde_json::to_value(build_base_spec( + sandbox, + config, + token_secret_name, + gpu_device_ids, + requested_image, + image_id, + oci_user, + supervisor_bin_path, + tls_secret_names, + )?) + .map_err(|error| ComputeDriverError::Message(format!("encode Podman spec: {error}"))) +} + +#[allow(clippy::too_many_arguments)] +fn build_base_spec( + sandbox: &DriverSandbox, + config: &PodmanComputeConfig, + token_secret_name: Option<&str>, + gpu_device_ids: Option<&[String]>, + requested_image: &str, + image_id: &str, + oci_user: &str, + supervisor_bin_path: Option<&Path>, + tls_secret_names: Option<&[String; 3]>, +) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); - let env = build_env(sandbox, config, requested_image, oci_user); + let env = build_env(sandbox, config, requested_image, oci_user)?; let labels = build_labels(sandbox); let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) @@ -1080,7 +1104,7 @@ pub fn build_container_spec_for_image( Vec::new() } else { vec![ImageVolume { - source: config.supervisor_image.clone(), + source: config.sandbox_runtime_image.clone(), destination: SUPERVISOR_MOUNT_DIR.into(), rw: false, }] @@ -1098,10 +1122,10 @@ pub fn build_container_spec_for_image( labels, env, volumes, - // Side-load the supervisor binary from a standalone OCI image. + // Side-load the sandbox runtime binary from its standalone OCI image. // Podman resolves image_volumes at the libpod layer, mounting the // image's filesystem at the destination path without starting a - // container from it. The supervisor image exposes the binary at + // container from it. The sandbox runtime image exposes the binary at // /openshell-sandbox, so it appears at /opt/openshell/bin/openshell-sandbox. image_volumes, hostname: format!("sandbox-{}", sandbox.name), @@ -1117,86 +1141,19 @@ pub fn build_container_spec_for_image( // corporate proxy flags follow it; the workload command comes from // the reserved environment variable. command, - // Force the supervisor to run as root (UID 0). Sandbox images may - // set a non-root USER directive (e.g. `USER sandbox`), but the - // supervisor needs root to create network namespaces, set up the - // proxy, and configure Landlock/seccomp. This matches the K8s - // driver's runAsUser: 0. - user: "0:0".into(), - // Podman's default container capability set is already restricted: - // CHOWN DAC_OVERRIDE FOWNER FSETID KILL SETGID SETUID SETPCAP - // NET_BIND_SERVICE SYS_CHROOT SETFCAP - // We add what the supervisor needs and drop what it doesn't. - cap_drop: vec![ - // Not needed: standard file permission bits are sufficient; dropping - // prevents the supervisor from bypassing DAC checks it shouldn't need. - "DAC_OVERRIDE".into(), - // Not needed: the supervisor does not create setuid/setgid executables. - "FSETID".into(), - // Not needed: the supervisor does not bind privileged ports (<1024). - "NET_BIND_SERVICE".into(), - // Not in Podman's default set but explicitly denied in case the image - // or runtime adds it; raw sockets are not required. - "NET_RAW".into(), - // Not needed: the supervisor does not manipulate file capabilities. - "SETFCAP".into(), - // Not needed: the supervisor does not call chroot(). - "SYS_CHROOT".into(), - ], - cap_add: vec![ - // seccomp filter installation, namespace creation, Landlock setup. - "SYS_ADMIN".into(), - // Network namespace veth setup, IP/route configuration. - "NET_ADMIN".into(), - // Reading /proc//exe and ancestor walk for process identity in policy. - "SYS_PTRACE".into(), - // Reading /dev/kmsg for bypass-detection diagnostics. - "SYSLOG".into(), - // Reading /proc//fd/ across UIDs for process identity resolution. - // In rootless Podman the supervisor runs as UID 0 inside a user namespace - // while sandbox processes run as the sandbox user. The kernel's - // proc_fd_permission() calls generic_permission() which denies cross-UID - // access to the dr-x------ fd directory unless this cap is present. - // Without it the proxy cannot determine which binary made each outbound - // connection and all traffic is denied. - "DAC_READ_SEARCH".into(), - // Child setup clears the capability bounding set before exec, which - // requires CAP_SETPCAP in the supervisor until drop_privileges(). - "SETPCAP".into(), - // Forwarding shutdown signals to the canonical workload process - // group after it drops to the sandbox UID requires CAP_KILL. - "KILL".into(), - ], - // SETUID, SETGID, SETPCAP, CHOWN, and FOWNER are intentionally kept from - // Podman's default set and not dropped: - // SETUID/SETGID – drop_privileges(): setuid()/setgid()/initgroups() to the - // sandbox user. In rootless Podman cap_drop:ALL removes them - // from the bounding set even though uid=0 owns the user - // namespace — so we keep them by not dropping them explicitly. - // SETPCAP – drop_privileges(): clears the child capability - // bounding set before the sandbox user execs. - // CHOWN – prepare_filesystem(): chown(path, uid, gid) on newly - // created read_write directories so the sandbox user can - // write to them. - // FOWNER – chown on files where the supervisor is not the owner - // (e.g. pre-existing directories owned by another user). - // - // Disable the container-level seccomp profile. The sandbox supervisor The sandbox supervisor - // installs its own policy-aware BPF seccomp filter at runtime via - // seccompiler (two-phase: clone3 blocker + main filter). The runtime - // filter is more restrictive than Podman's default — it blocks 20+ - // dangerous syscalls and conditionally restricts socket domains based - // on network policy. The filter self-seals by blocking further - // seccomp(SET_MODE_FILTER) calls after installation. - // - // A container-level profile would interfere by blocking the landlock - // and seccomp syscalls the supervisor needs during setup, before it - // locks itself down. + // The paired builder supplies the immutable non-root identity. + user: String::new(), + groups: Vec::new(), + unsetenv: Vec::new(), + cap_drop: vec!["ALL".into()], + cap_add: Vec::new(), no_new_privileges: true, - seccomp_profile_path: "unconfined".into(), - apparmor_profile: podman_apparmor_profile(config.app_armor_profile.as_ref()), + apparmor_profile: None, + // Omission selects the runtime default, never an unconfined profile. + seccomp_profile_path: String::new(), + sysctl: BTreeMap::new(), image_pull_policy: "never".to_string(), - healthconfig: config.health_check_interval_secs.map(|interval_secs| HealthConfig { + healthconfig: HealthConfig { test: vec![ "CMD-SHELL".into(), format!( @@ -1205,11 +1162,14 @@ pub fn build_container_spec_for_image( openshell_core::config::DEFAULT_SSH_PORT ), ], - interval: interval_secs.get() * 1_000_000_000, + interval: config + .health_check_interval_secs + .map_or(10, std::num::NonZeroU64::get) + * 1_000_000_000, timeout: 2_000_000_000, retries: 10, start_period: 5_000_000_000, - }), + }, resource_limits, secret_env: BTreeMap::new(), secrets: { @@ -1409,7 +1369,211 @@ pub fn build_container_spec_for_image( }, }; - Ok(serde_json::to_value(container_spec).expect("ContainerSpec serialization cannot fail")) + Ok(container_spec) +} + +/// Driver-owned inputs for the two independent runtime containers. +pub struct IsolationSpecInput<'a> { + pub sandbox: &'a DriverSandbox, + pub config: &'a PodmanComputeConfig, + pub token_secret: Option<&'a str>, + pub gpu_devices: Option<&'a [String]>, + pub requested_image: &'a str, + pub image_id: &'a str, + pub image_user: &'a str, + pub image_env: &'a [String], + pub supervisor_bin: Option<&'a Path>, + pub tls_secrets: Option<&'a [String; 3]>, + pub identity: &'a openshell_isolation_interface::contract::ResolvedWorkloadIdentity, +} + +pub struct IsolationSpecs { + pub workload: ContainerSpec, + pub supervisor: ContainerSpec, +} + +pub fn build_isolation_specs( + input: IsolationSpecInput<'_>, +) -> Result { + let base = || { + build_base_spec( + input.sandbox, + input.config, + input.token_secret, + input.gpu_devices, + input.requested_image, + input.image_id, + input.image_user, + input.supervisor_bin, + input.tls_secrets, + ) + }; + let mut workload = base()?; + let mut supervisor = base()?; + let user = format!("{}:{}", input.identity.uid, input.identity.gid); + let channel = crate::isolation::channel_volume_name(&input.sandbox.id); + + workload + .labels + .insert(crate::isolation::LABEL_ROLE.into(), "sandbox".into()); + workload.env = BTreeMap::new(); + workload.unsetenv = input + .image_env + .iter() + .filter_map(|entry| entry.split_once('=').map(|(key, _)| key.to_string())) + .collect(); + workload.command = vec![ + "--bootstrap".into(), + crate::isolation::BOOTSTRAP_PATH.into(), + ]; + workload.user.clone_from(&user); + workload.groups = input + .identity + .supplementary_gids + .iter() + .map(ToString::to_string) + .collect(); + workload.cap_drop = vec!["ALL".into()]; + workload.cap_add.clear(); + workload.apparmor_profile = input + .config + .app_armor_profile + .as_ref() + .and_then(openshell_core::config::AppArmorProfile::oci_security_opt) + .and_then(|option| option.strip_prefix("apparmor=").map(str::to_string)); + workload.seccomp_profile_path.clear(); + workload + .sysctl + .insert("net.ipv4.ip_unprivileged_port_start".into(), "0".into()); + workload.netns.nsmode = "none".into(); + workload.networks.clear(); + workload.portmappings.clear(); + workload.hostadd.clear(); + workload.secret_env.clear(); + workload.secrets.clear(); + workload.healthconfig.test = vec!["NONE".into()]; + workload + .mounts + .retain(|mount| !trusted_mount(&mount.destination)); + workload.mounts.push(Mount { + kind: "tmpfs".into(), + source: "tmpfs".into(), + destination: openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT.into(), + options: vec![ + "rw".into(), + "noexec".into(), + "nosuid".into(), + "nodev".into(), + // Libpod's OCI `mounts` API rejects tmpfs uid/gid options. The + // unprivileged runtime creates the owned material subdirectory. + "mode=0777".into(), + "size=1m".into(), + ], + }); + workload.volumes.push(NamedVolume { + name: channel.clone(), + dest: crate::isolation::CHANNEL_ROOT.into(), + options: vec!["rw".into(), "nocopy".into(), "z".into()], + }); + + supervisor.name = crate::isolation::supervisor_name(&input.sandbox.id); + supervisor + .labels + .insert(crate::isolation::LABEL_ROLE.into(), "supervisor".into()); + supervisor.image.clone_from(&input.config.supervisor_image); + supervisor.entrypoint = vec!["/openshell-supervisor".into()]; + supervisor.command.extend([ + format!( + "--backend-descriptor-file={}", + crate::isolation::RUNTIME_DESCRIPTOR_PATH + ), + format!("--auth-bundle-file={}", crate::isolation::AUTH_BUNDLE_PATH), + "--health-socket-path=/run/openshell/supervisor-health.sock".into(), + ]); + supervisor.env.insert( + openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND.into(), + openshell_sandbox_backend::BACKEND_NAME.into(), + ); + supervisor.user = user; + supervisor.groups = input + .identity + .supplementary_gids + .iter() + .map(ToString::to_string) + .collect(); + supervisor.cap_drop = vec!["ALL".into()]; + supervisor.cap_add.clear(); + supervisor.seccomp_profile_path.clear(); + // The trusted supervisor originates approved egress from the Podman host + // network. Keep it in the caller's user namespace as well: joining the + // workload's user namespace is incompatible with host networking under + // rootless Podman, and the authenticated channel does not require a shared + // user namespace. The workload remains fenced by network=none. + supervisor.userns = Some(UserNS { + nsmode: "host".into(), + value: None, + }); + supervisor.idmappings = None; + supervisor.netns.nsmode = "host".into(); + supervisor.networks.clear(); + supervisor.portmappings.clear(); + supervisor.devices = None; + supervisor.image_volumes.clear(); + supervisor.volumes = vec![NamedVolume { + name: channel, + dest: crate::isolation::CHANNEL_ROOT.into(), + options: vec!["ro".into(), "nocopy".into(), "z".into()], + }]; + supervisor.mounts.retain(|mount| { + trusted_mount(&mount.destination) + && mount.destination != openshell_core::container_paths::NETNS_MOUNT_ROOT + }); + for destination in ["/run", "/var/log", "/tmp"] { + supervisor.mounts.push(Mount { + kind: "tmpfs".into(), + source: "tmpfs".into(), + destination: destination.into(), + options: vec![ + "rw".into(), + "noexec".into(), + "nosuid".into(), + "nodev".into(), + "mode=0777".into(), + "size=64m".into(), + ], + }); + } + for secret in &mut supervisor.secrets { + secret.uid = input.identity.uid; + secret.gid = input.identity.gid; + } + if let Some(interval) = input.config.health_check_interval_secs { + supervisor.healthconfig.test = vec![ + "CMD".into(), + "/openshell-supervisor".into(), + "health".into(), + "--socket".into(), + "/run/openshell/supervisor-health.sock".into(), + ]; + supervisor.healthconfig.interval = interval.get() * 1_000_000_000; + } else { + supervisor.healthconfig.test = vec!["NONE".into()]; + } + Ok(IsolationSpecs { + workload, + supervisor, + }) +} + +fn trusted_mount(destination: &str) -> bool { + matches!( + destination, + TLS_CA_MOUNT_PATH + | TLS_CERT_MOUNT_PATH + | TLS_KEY_MOUNT_PATH + | PROXY_CA_MOUNT_PATH + | PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR + ) || destination == openshell_core::container_paths::NETNS_MOUNT_ROOT } fn provider_spiffe_workload_api_socket_env_value(config: &PodmanComputeConfig) -> Option { @@ -1516,6 +1680,122 @@ mod tests { static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + #[test] + fn isolated_pair_keeps_privileges_network_and_secrets_out_of_workload() { + let sandbox = DriverSandbox { + id: "pair".into(), + name: "agent".into(), + ..Default::default() + }; + let mut config = PodmanComputeConfig::default(); + config.app_armor_profile = Some(openshell_core::config::AppArmorProfile::Localhost( + "openshell-sandbox".into(), + )); + config.userns = Some("auto".into()); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![2000], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let env = vec![ + "LD_PRELOAD=/hostile.so".into(), + "HTTP_PROXY=http://bypass".into(), + ]; + let specs = build_isolation_specs(IsolationSpecInput { + sandbox: &sandbox, + config: &config, + token_secret: Some("jwt"), + gpu_devices: None, + requested_image: "image:latest", + image_id: "sha256:image", + image_user: "1000:1001", + image_env: &env, + supervisor_bin: None, + tls_secrets: None, + identity: &identity, + }) + .unwrap(); + for spec in [&specs.workload, &specs.supervisor] { + assert_eq!(spec.user, "1000:1001"); + assert_eq!(spec.groups, vec!["2000"]); + assert_eq!(spec.cap_drop, vec!["ALL"]); + assert!(spec.cap_add.is_empty()); + assert!(spec.seccomp_profile_path.is_empty()); + assert!(spec.no_new_privileges); + } + assert_eq!(specs.workload.netns.nsmode, "none"); + assert_eq!( + specs + .workload + .userns + .as_ref() + .map(|userns| userns.nsmode.as_str()), + Some("auto") + ); + assert_eq!( + specs.workload.apparmor_profile.as_deref(), + Some("openshell-sandbox") + ); + assert_eq!(specs.supervisor.apparmor_profile, None); + let workload_json = serde_json::to_string(&specs.workload).unwrap(); + assert!(workload_json.contains("\"apparmor_profile\":\"openshell-sandbox\"")); + assert_eq!(specs.supervisor.healthconfig.test, vec!["NONE"]); + assert!(specs.workload.networks.is_empty()); + assert!(specs.workload.portmappings.is_empty()); + assert_eq!(specs.supervisor.netns.nsmode, "host"); + assert_eq!( + specs + .supervisor + .userns + .as_ref() + .map(|userns| userns.nsmode.as_str()), + Some("host") + ); + assert!(specs.supervisor.idmappings.is_none()); + assert!(specs.supervisor.networks.is_empty()); + assert!(specs.supervisor.portmappings.is_empty()); + assert!(specs.workload.env.is_empty()); + assert_eq!(specs.workload.unsetenv, vec!["LD_PRELOAD", "HTTP_PROXY"]); + assert!(specs.workload.secrets.is_empty()); + assert!( + specs + .workload + .mounts + .iter() + .all(|mount| !trusted_mount(&mount.destination)) + ); + let supervisor_ca_mount = specs + .workload + .mounts + .iter() + .find(|mount| { + mount.destination == openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT + }) + .expect("workload supervisor CA mount"); + assert_eq!(supervisor_ca_mount.kind, "tmpfs"); + assert_eq!(supervisor_ca_mount.source, "tmpfs"); + for option in ["rw", "noexec", "nosuid", "nodev", "mode=0777", "size=1m"] { + assert!(supervisor_ca_mount.options.contains(&option.to_string())); + } + assert!( + specs + .workload + .mounts + .iter() + .all(|mount| mount.destination != "/run") + ); + assert_eq!(specs.supervisor.secrets.len(), 1); + assert_eq!(specs.supervisor.secrets[0].source, "jwt"); + assert_eq!(specs.supervisor.secrets[0].uid, 1000); + assert_eq!(specs.supervisor.volumes.len(), 1); + assert!(specs.supervisor.volumes[0].options.contains(&"ro".into())); + assert!(specs.supervisor.volumes[0].options.contains(&"z".into())); + assert_eq!(specs.supervisor.entrypoint, vec!["/openshell-supervisor"]); + } + fn json_struct(value: Value) -> prost_types::Struct { let Value::Object(object) = value else { panic!("expected JSON object"); @@ -1564,7 +1844,7 @@ mod tests { } #[test] - fn container_spec_applies_resource_limits() { + fn container_spec_applies_cpu_and_memory_limits() { use openshell_core::proto::compute::v1::{ DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, }; @@ -1581,22 +1861,18 @@ mod tests { }), ..Default::default() }); - let mut config = test_config(); - config.sandbox_pids_limit = std::num::NonZeroI64::new(2048); - let spec = build_container_spec(&sandbox, &config); + let config = test_config(); + let limits = build_resource_limits(&sandbox, &config); + assert_eq!(limits.cpu.quota, 50_000); + assert_eq!(limits.memory.limit, 2 * 1024 * 1024 * 1024); assert_eq!( - spec["resource_limits"]["cpu"]["quota"].as_u64(), - Some(50_000) - ); - assert_eq!( - spec["resource_limits"]["memory"]["limit"].as_u64(), - Some(2 * 1024 * 1024 * 1024) - ); - assert_eq!( - spec["resource_limits"]["pids"]["limit"].as_i64(), - Some(2048) + limits.pids.as_ref().map(|pids| pids.limit), + openshell_core::config::default_sandbox_pids_limit().map(std::num::NonZeroI64::get) ); + let serialized = serde_json::to_string(&limits).unwrap(); + assert!(serialized.contains("\"pids\":{\"limit\":2048}")); + assert!(!serialized.contains("PidsLimit")); } #[test] @@ -1604,43 +1880,9 @@ mod tests { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); config.sandbox_pids_limit = None; - let spec = build_container_spec(&sandbox, &config); + let limits = build_resource_limits(&sandbox, &config); - assert!(spec["resource_limits"].get("pids").is_none()); - } - - #[test] - fn container_spec_uses_podman_apparmor_profile_field() { - let sandbox = test_sandbox("test-id", "test-name"); - - for (profile, expected) in [ - (openshell_core::AppArmorProfile::Unconfined, "unconfined"), - ( - openshell_core::AppArmorProfile::Localhost("openshell-supervisor".to_string()), - "openshell-supervisor", - ), - ] { - let mut config = test_config(); - config.app_armor_profile = Some(profile); - let spec = build_container_spec(&sandbox, &config); - - assert_eq!(spec["apparmor_profile"].as_str(), Some(expected)); - assert!(spec.get("security_opt").is_none()); - } - } - - #[test] - fn container_spec_omits_podman_apparmor_profile_for_runtime_default() { - let sandbox = test_sandbox("test-id", "test-name"); - - for profile in [None, Some(openshell_core::AppArmorProfile::RuntimeDefault)] { - let mut config = test_config(); - config.app_armor_profile = profile; - let spec = build_container_spec(&sandbox, &config); - - assert!(spec.get("apparmor_profile").is_none()); - assert!(spec.get("security_opt").is_none()); - } + assert!(limits.pids.is_none()); } #[test] @@ -1681,7 +1923,8 @@ mod tests { container["env"]["OPENSHELL_CONTAINER_IMAGE"].as_str(), Some("registry.example/app:latest") ); - assert_eq!(container["user"].as_str(), Some("0:0")); + // Only the paired builder materializes the immutable non-root user. + assert_eq!(container["user"].as_str(), Some("")); assert_eq!(container["image_pull_policy"].as_str(), Some("never")); assert_eq!(container["dns_search"], serde_json::json!([])); assert_eq!(container["dns_option"], serde_json::json!([])); @@ -1921,66 +2164,25 @@ mod tests { } #[test] - fn container_spec_includes_required_capabilities() { + fn container_spec_defaults_drop_capabilities_and_keep_runtime_seccomp() { let sandbox = test_sandbox("test-id", "test-name"); let config = test_config(); - let spec = build_container_spec(&sandbox, &config); - - let added: Vec<&str> = spec["cap_add"] - .as_array() - .expect("cap_add should be an array") - .iter() - .filter_map(|v| v.as_str()) - .collect(); - assert!(added.contains(&"SYS_ADMIN"), "missing SYS_ADMIN"); - assert!(added.contains(&"NET_ADMIN"), "missing NET_ADMIN"); - assert!(added.contains(&"SYS_PTRACE"), "missing SYS_PTRACE"); - assert!(added.contains(&"SYSLOG"), "missing SYSLOG"); - assert!( - added.contains(&"DAC_READ_SEARCH"), - "missing DAC_READ_SEARCH" - ); - assert!(added.contains(&"SETPCAP"), "missing SETPCAP"); - assert!(added.contains(&"KILL"), "missing KILL"); - - // SETUID and SETGID are NOT in cap_add — they remain available from the - // default bounding set because we no longer use cap_drop:ALL. Verify they - // are also not explicitly dropped. Similarly SETPCAP, CHOWN and FOWNER - // must not be dropped because child setup clears the bounding set and - // prepare_filesystem() calls chown() on newly created read_write - // directories before the supervisor drops privileges. - let dropped: Vec<&str> = spec["cap_drop"] - .as_array() - .expect("cap_drop should be an array") - .iter() - .filter_map(|v| v.as_str()) - .collect(); - assert!(!dropped.contains(&"SETUID"), "SETUID must not be dropped"); - assert!(!dropped.contains(&"SETGID"), "SETGID must not be dropped"); - assert!( - dropped.contains(&"NET_BIND_SERVICE"), - "NET_BIND_SERVICE must stay dropped; policy DNS binds an unprivileged port" - ); - assert!( - !dropped.contains(&"CHOWN"), - "CHOWN must not be dropped (needed for prepare_filesystem chown)" - ); - assert!( - !dropped.contains(&"FOWNER"), - "FOWNER must not be dropped (needed for chown on non-owned files)" - ); - assert!( - !dropped.contains(&"SETPCAP"), - "SETPCAP must not be dropped (needed for child bounding-set clear)" - ); - assert!( - !dropped.contains(&"KILL"), - "KILL must not be dropped (needed to signal the sandbox workload on shutdown)" - ); - assert!( - !dropped.contains(&"ALL"), - "must not use cap_drop:ALL in rootless Podman" - ); + let spec = build_base_spec( + &sandbox, + &config, + None, + None, + "image", + "sha256:image", + "", + None, + None, + ) + .unwrap(); + assert_eq!(spec.cap_drop, vec!["ALL"]); + assert!(spec.cap_add.is_empty()); + assert!(spec.seccomp_profile_path.is_empty()); + assert!(spec.no_new_privileges); } #[test] @@ -2016,8 +2218,7 @@ mod tests { #[test] fn container_spec_healthcheck_accepts_supervisor_socket() { let sandbox = test_sandbox("test-id", "test-name"); - let mut config = test_config(); - config.health_check_interval_secs = std::num::NonZeroU64::new(10); + let config = test_config(); let spec = build_container_spec(&sandbox, &config); let healthcheck = spec["healthconfig"]["test"] @@ -2033,13 +2234,6 @@ mod tests { ); } - #[test] - fn container_spec_omits_healthcheck_when_disabled() { - let sandbox = test_sandbox("test-id", "test-name"); - let spec = build_container_spec(&sandbox, &test_config()); - assert!(spec.get("healthconfig").is_none()); - } - #[test] fn container_spec_healthcheck_interval_from_config() { let sandbox = test_sandbox("test-id", "test-name"); @@ -2476,7 +2670,7 @@ mod tests { } #[test] - fn container_spec_includes_supervisor_image_volume() { + fn container_spec_includes_sandbox_runtime_image_volume() { let sandbox = test_sandbox("test-id", "test-name"); let config = test_config(); let spec = build_container_spec(&sandbox, &config); @@ -2493,8 +2687,8 @@ mod tests { let vol = &image_volumes[0]; assert_eq!( vol["source"].as_str(), - Some(openshell_core::config::default_supervisor_image().as_str()), - "image volume source should be the supervisor image" + Some(openshell_core::config::default_sandbox_runtime_image().as_str()), + "image volume source should be the sandbox runtime image" ); assert_eq!( vol["destination"].as_str(), @@ -2579,9 +2773,9 @@ mod tests { let image_volumes = spec["image_volumes"] .as_array() .expect("image_volumes should be an array"); - let expected_supervisor = openshell_core::config::default_supervisor_image(); + let expected_sandbox_runtime = openshell_core::config::default_sandbox_runtime_image(); assert!(image_volumes.iter().any(|volume| { - volume["source"].as_str() == Some(expected_supervisor.as_str()) + volume["source"].as_str() == Some(expected_sandbox_runtime.as_str()) && volume["destination"].as_str() == Some("/opt/openshell/bin") })); assert!(image_volumes.iter().any(|volume| { @@ -2930,6 +3124,27 @@ mod tests { assert!(err.to_string().contains("reserved OpenShell path")); } + #[test] + fn user_mounts_cannot_replace_private_channel_hierarchy() { + for target in [ + "/.openshell", + "/.openshell/channel", + "/.openshell/channel/sandbox", + "/.openshell/supervisor", + ] { + let mount = PodmanDriverMountConfig::Volume { + source: "user-owned".into(), + target: target.into(), + read_only: false, + subpath: None, + }; + assert!( + validate_podman_driver_mounts(&[mount], false).is_err(), + "{target}" + ); + } + } + #[test] fn container_spec_uses_configured_host_gateway_ip() { let sandbox = test_sandbox("test-id", "test-name"); @@ -3367,7 +3582,7 @@ mod tests { !image_volumes .iter() .any(|v| v["destination"].as_str() == Some(SUPERVISOR_MOUNT_DIR)), - "supervisor image volume should not be present when bind path is provided" + "sandbox runtime image volume should not be present when bind path is provided" ); let mounts = spec["mounts"] @@ -3401,7 +3616,7 @@ mod tests { image_volumes .iter() .any(|v| v["destination"].as_str() == Some(SUPERVISOR_MOUNT_DIR)), - "supervisor image volume should be present by default" + "sandbox runtime image volume should be present by default" ); let mounts = spec["mounts"] diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index cea9268fef..1b9df795dd 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -13,7 +13,7 @@ use crate::watcher::{ use openshell_core::ComputeDriverError; use openshell_core::config::CDI_GPU_DEVICE_ALL; use openshell_core::driver_utils::{ - GatewayCallbackRoute, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + GatewayCallbackRoute, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, }; @@ -30,6 +30,7 @@ use openshell_core::proto::compute::v1::{ GpuResourceCapabilities, GpuResourceRequirements, MemoryResourceCapabilities, ResourceCapabilities, gateway_listener_requirement::Selector, }; +use std::collections::HashMap; #[cfg(target_os = "linux")] use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; @@ -41,6 +42,24 @@ use url::Url; const STOP_COMPLETION_POLL_INTERVAL: Duration = Duration::from_millis(50); const STOP_COMPLETION_TIMEOUT_HEADROOM: Duration = Duration::from_secs(5); +fn decode_launch_authentication( + encoded: &[u8], +) -> Result { + let authentication = + serde_json::from_slice::(encoded) + .map_err(|error| { + ComputeDriverError::Precondition(format!( + "decode Podman sandbox launch authentication: {error}" + )) + })?; + authentication.validate().map_err(|error| { + ComputeDriverError::Precondition(format!( + "validate Podman sandbox launch authentication: {error}" + )) + })?; + Ok(authentication) +} + impl From for ComputeDriverError { fn from(value: PodmanApiError) -> Self { match value { @@ -437,7 +456,7 @@ impl PodmanComputeDriver { } // Auto-detect the gRPC callback endpoint before deciding whether this - // topology needs the Podman bridge gateway address. + // callback route needs the Podman bridge gateway address. if config.grpc_endpoint.is_empty() { config.grpc_endpoint = gateway_callback_endpoint( GatewayCallbackRoute::Podman, @@ -452,7 +471,7 @@ impl PodmanComputeDriver { } // Ensure the bridge network exists. Inspect its gateway only when the - // selected Linux callback topology will bind that exact address. + // selected Linux callback route will bind that exact address. client.ensure_network(&config.network_name).await?; let uses_local_callback_alias = Url::parse(&config.grpc_endpoint) .ok() @@ -777,13 +796,29 @@ impl PodmanComputeDriver { "Creating sandbox container" ); - let (image, immutable_image_id, image_user) = async { + let (image, immutable_image_id, image_user, image_env) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { - // The supervisor binary is shipped in a standalone OCI image and - // mounted into sandbox containers via Podman's type=image mount. + // The sandbox runtime is shipped in a standalone OCI image. + // The driver extracts and verifies its binary before bind-mounting + // it into the workload container. + let sandbox_runtime_pull_policy = + runtime_image_pull_policy(&self.config.sandbox_runtime_image); + info!( + image = %self.config.sandbox_runtime_image, + policy = sandbox_runtime_pull_policy, + "Ensuring sandbox runtime image" + ); + self.client + .pull_image( + &self.config.sandbox_runtime_image, + sandbox_runtime_pull_policy, + ) + .await + .map_err(ComputeDriverError::from)?; + let supervisor_pull_policy = - supervisor_image_pull_policy(&self.config.supervisor_image); + runtime_image_pull_policy(&self.config.supervisor_image); info!( image = %self.config.supervisor_image, policy = supervisor_pull_policy, @@ -837,7 +872,8 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; } - Ok((image.to_string(), inspected_image.id, image_user)) + let image_env = inspected_image.config.as_ref().map_or_else(Vec::new, |config| config.env.clone()); + Ok((image.to_string(), inspected_image.id, image_user, image_env)) } .await; phase_status.finish(result) @@ -856,6 +892,32 @@ impl PodmanComputeDriver { // content at startup. validate_sandbox_proxy_ca_bundle(&self.config).await?; + let identity = self + .resolve_workload_identity(sandbox, &immutable_image_id, &image_user) + .await?; + let channel_volume = crate::isolation::channel_volume_name(&sandbox.id); + let mut runtime_config = self.config.clone(); + runtime_config.sandbox_runtime_image = self + .client + .inspect_image(&self.config.sandbox_runtime_image) + .await? + .id; + if runtime_config.sandbox_runtime_image.is_empty() { + return Err(ComputeDriverError::Precondition( + "sandbox runtime image inspection returned no immutable image ID".into(), + )); + } + runtime_config.supervisor_image = self + .client + .inspect_image(&self.config.supervisor_image) + .await? + .id; + if runtime_config.supervisor_image.is_empty() { + return Err(ComputeDriverError::Precondition( + "supervisor image inspection returned no immutable image ID".into(), + )); + } + // Create workspace volume and per-sandbox token secret. let (token_secret_name, proxy_auth_secret_name) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); @@ -901,6 +963,7 @@ impl PodmanComputeDriver { // Clean up the volume and both per-sandbox secrets on any failure past // this point. let cleanup_created = || async { + let _ = self.client.remove_volume(&channel_volume).await; let _ = self.client.remove_volume(&vol_name).await; if let Some(secret) = token_secret_name.as_deref() { cleanup_sandbox_token_secret(&self.client, secret).await; @@ -911,7 +974,7 @@ impl PodmanComputeDriver { }; // Prepare and create the container. - let tls_secret_names = async { + async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { let gpu_devices = match self.resolve_gpu_cdi_devices( @@ -925,22 +988,19 @@ impl PodmanComputeDriver { return Err(e); } }; - let supervisor_bin_path = if userns_needs_extraction(self.config.userns.as_deref()) - { - match extract_supervisor_bin(&self.client, &self.config).await { + // Podman's image-volume support varies across libpod/runtime + // combinations. Always use the verified extraction cache so + // workload startup does not depend on type=image mounts. + let supervisor_bin_path = + match extract_sandbox_bin(&self.client, &runtime_config).await { Ok(path) => Some(path), Err(e) => { cleanup_created().await; return Err(e); } - } - } else { - None - }; + }; - let tls_secret_names = if userns_remaps_uids(self.config.userns.as_deref()) - && self.config.tls_enabled() - { + let tls_secret_names = if self.config.tls_enabled() { let names = container::tls_secret_names(&sandbox.id); if let Err(e) = create_tls_secrets(&self.client, &self.config, &names).await { cleanup_created().await; @@ -958,32 +1018,90 @@ impl PodmanComputeDriver { } }; - let spec = match container::build_container_spec_for_image( + let specs = container::build_isolation_specs(container::IsolationSpecInput { sandbox, - &self.config, - token_secret_name.as_deref(), - gpu_devices.as_deref(), - &image, - &immutable_image_id, - &image_user, - supervisor_bin_path.as_deref(), - tls_secret_names.as_ref(), - ) { + config: &runtime_config, + token_secret: token_secret_name.as_deref(), + gpu_devices: gpu_devices.as_deref(), + requested_image: &image, + image_id: &immutable_image_id, + image_user: &image_user, + image_env: &image_env, + supervisor_bin: supervisor_bin_path.as_deref(), + tls_secrets: tls_secret_names.as_ref(), + identity: &identity, + }); + let specs = match specs { Ok(spec) => spec, Err(e) => { cleanup_all().await; return Err(e); } }; - match self.client.create_container(&spec).await { - Ok(_) => Ok(tls_secret_names), - Err(PodmanApiError::Conflict(_)) => { - cleanup_all().await; - Err(ComputeDriverError::AlreadyExists) + let mut created_workload = None; + let mut created_supervisor = None; + let create_result = async { + self.client.create_volume(&channel_volume).await?; + let workload_id = self.client.create_typed_container(&specs.workload).await?; + created_workload = Some(workload_id.clone()); + self.client.verify_isolation_fence(&workload_id).await?; + let child_env = podman_child_environment(sandbox, &image_env); + let launch_authentication = sandbox + .spec + .as_ref() + .filter(|spec| !spec.launch_authentication.is_empty()) + .ok_or_else(|| { + ComputeDriverError::Precondition( + "Podman sandbox launch authentication is required".to_string(), + ) + }) + .and_then(|spec| { + decode_launch_authentication(&spec.launch_authentication) + })?; + let archives = crate::isolation::bootstrap_archives( + &sandbox.id, + &workload_id, + &uuid::Uuid::new_v4().to_string(), + &identity, + child_env, + &launch_authentication, + )?; + self.client + .copy_to_container( + &workload_id, + crate::isolation::CHANNEL_ROOT, + archives.channel, + ) + .await?; + self.client + .copy_to_container(&workload_id, "/sandbox", archives.workspace) + .await?; + let supervisor_id = self + .client + .create_typed_container(&specs.supervisor) + .await?; + created_supervisor = Some(supervisor_id.clone()); + self.client + .copy_to_container(&supervisor_id, "/", archives.supervisor) + .await?; + // Start the sandbox only after both containers and their + // bootstrap material exist. It keeps the agent stopped + // until the authenticated supervisor confirms the boundary. + self.client.start_container(&workload_id).await?; + self.client.start_container(&supervisor_id).await?; + Ok::<(), ComputeDriverError>(()) + } + .await; + if create_result.is_err() { + for id in [created_supervisor, created_workload].into_iter().flatten() { + let _ = self.client.remove_container(&id, 0).await; } + } + match create_result { + Ok(()) => Ok(()), Err(e) => { cleanup_all().await; - Err(ComputeDriverError::from(e)) + Err(e) } } } @@ -998,44 +1116,6 @@ impl PodmanComputeDriver { )) .await?; - let cleanup_all = || async { - cleanup_created().await; - if let Some(names) = &tls_secret_names { - cleanup_tls_secrets(&self.client, names).await; - } - }; - - // Start container. - let start_result = async { - let phase_status = openshell_otel::ErrorStatusGuard::current(); - let result = self - .client - .start_container(&name) - .await - .map_err(ComputeDriverError::from); - phase_status.finish(result) - } - .instrument(tracing::info_span!( - "podman.start_container", - otel.name = "podman.start_container", - otel.status_code = tracing::field::Empty, - container.name = %name, - )) - .await; - if let Err(e) = start_result { - warn!( - sandbox_name = %sandbox.name, - error = %e, - "Failed to start container; cleaning up" - ); - let _ = self - .client - .remove_container(&name, self.config.stop_timeout_secs) - .await; - cleanup_all().await; - return Err(e); - } - info!( sandbox_id = %sandbox.id, sandbox_name = %sandbox.name, @@ -1045,7 +1125,65 @@ impl PodmanComputeDriver { span_status.finish(Ok(())) } - /// Find the Podman container ID for a sandbox by its sandbox ID using label lookup. + /// Resolve image accounts without executing any image-supplied program. + async fn resolve_workload_identity( + &self, + sandbox: &DriverSandbox, + image: &str, + image_user: &str, + ) -> Result + { + #[derive(serde::Serialize)] + struct InspectionSpec<'a> { + name: String, + image: &'a str, + } + // Inspect a stopped, unexecuted container pinned to the final image ID. + let id = self + .client + .create_typed_container(&InspectionSpec { + name: format!("openshell-identity-{}", uuid::Uuid::new_v4()), + image, + }) + .await?; + let result = + async { + // Do not let image-controlled symlinks alias the protected channel + // into an agent-readable subtree before Podman mounts it. + match self.client.copy_from_container(&id, "/.openshell").await { + Err(PodmanApiError::NotFound(_)) => {} + Ok(_) => return Err(ComputeDriverError::Precondition( + "workload images must not prepopulate the reserved /.openshell hierarchy" + .into(), + )), + Err(error) => return Err(error.into()), + } + let mut accounts = Vec::new(); + for path in ["/etc/passwd", "/etc/group"] { + let content = match self.client.copy_from_container(&id, path).await { + Ok(archive) => extract_first_tar_entry(&archive) + .map_err(ComputeDriverError::Precondition)?, + Err(PodmanApiError::NotFound(_)) => Vec::new(), + Err(error) => return Err(error.into()), + }; + accounts.push(content); + } + let [passwd, group] = accounts.as_slice() else { + return Err(ComputeDriverError::Precondition( + "image account inspection was incomplete".into(), + )); + }; + crate::isolation::resolve_identity(sandbox, image, image_user, passwd, group) + } + .await; + let cleanup = self.client.remove_container(&id, 0).await; + if let Err(error) = cleanup { + warn!(container = %id, %error, "Failed to remove stopped identity inspection container"); + } + result + } + + /// Find only the workload, never its supervisor companion. async fn find_container_id( &self, sandbox_id: &str, @@ -1060,7 +1198,11 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; Ok(entries.into_iter().next()) @@ -1113,6 +1255,15 @@ impl PodmanComputeDriver { .await? .ok_or(ComputeDriverError::NotFound)?; let container_id = container.id; + let supervisor = crate::isolation::supervisor_name(sandbox_id); + match self + .client + .stop_container(&supervisor, self.config.stop_timeout_secs) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error.into()), + } if container.state == "stopping" { let result = async { let finished_at = self @@ -1169,14 +1320,57 @@ impl PodmanComputeDriver { sandbox.id = %sandbox_id, ) )] - pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), ComputeDriverError> { + pub async fn start_sandbox( + &self, + sandbox_id: &str, + generation_id: &str, + encoded_authentication: &[u8], + ) -> Result<(), ComputeDriverError> { let span_status = openshell_otel::ErrorStatusGuard::current(); + let generation = openshell_core::sandbox_generation::SandboxGenerationId::parse( + generation_id.to_string(), + ) + .map_err(|error| ComputeDriverError::InvalidArgument(error.to_string()))?; + let launch_authentication = decode_launch_authentication(encoded_authentication)?; let container = self .find_container(sandbox_id) .await? .ok_or(ComputeDriverError::NotFound)?; if container.state == "running" { - return span_status.finish(Ok(())); + let supervisor = self + .client + .inspect_container(&crate::isolation::supervisor_name(sandbox_id)) + .await; + if supervisor + .as_ref() + .is_ok_and(|inspect| inspect.state.running) + { + let archive = self + .client + .copy_from_container( + &crate::isolation::supervisor_name(sandbox_id), + crate::isolation::RESTART_METADATA_PATH, + ) + .await?; + let bundle = + extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; + let metadata = crate::isolation::restart_metadata_from_slice(&bundle)?; + if metadata.generation == generation.as_str() && encoded_authentication.is_empty() { + return span_status.finish(Ok(())); + } + if metadata.generation != generation.as_str() { + return span_status.finish(Err(ComputeDriverError::Precondition(format!( + "Podman sandbox is already running generation {}", + metadata.generation + )))); + } + // A non-empty bundle for the same generation comes from + // gateway startup recovery. Restart both containers so the + // in-memory launch session changes atomically on both sides. + } + self.client.stop_container(&container.id, 0).await?; + self.wait_for_container_stopped(sandbox_id, &container.id) + .await?; } let container_id = container.id; info!(sandbox_id = %sandbox_id, container = %container_id, "Starting sandbox container"); @@ -1193,11 +1387,47 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; self.lifecycle_event_fences .record_previous_exit(sandbox_id, previous.state.finished_at.as_deref()); - let result = self - .client - .start_container(&container_id) - .await - .map_err(ComputeDriverError::from); + let result = async { + let supervisor = crate::isolation::supervisor_name(sandbox_id); + self.client + .stop_container(&supervisor, self.config.stop_timeout_secs) + .await?; + self.wait_for_container_stopped(sandbox_id, &supervisor) + .await?; + let archive = self + .client + .copy_from_container(&supervisor, crate::isolation::RESTART_METADATA_PATH) + .await?; + let bundle = + extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; + let restart_metadata = crate::isolation::restart_metadata_from_slice(&bundle)?; + let archives = crate::isolation::bootstrap_archives( + sandbox_id, + &container_id, + generation.as_str(), + &restart_metadata.workload_identity, + restart_metadata.child_env, + &launch_authentication, + )?; + self.client + .copy_to_container( + &container_id, + crate::isolation::CHANNEL_ROOT, + archives.channel, + ) + .await?; + self.client + .copy_to_container(&supervisor, "/", archives.supervisor) + .await?; + self.client.verify_isolation_fence(&container_id).await?; + self.client.start_container(&container_id).await?; + if let Err(error) = self.client.start_container(&supervisor).await { + let _ = self.client.stop_container(&container_id, 0).await; + return Err(error.into()); + } + Ok(()) + } + .await; span_status.finish(result) } @@ -1219,6 +1449,25 @@ impl PodmanComputeDriver { )); } + let supervisor = crate::isolation::supervisor_name(sandbox_id); + match self + .client + .remove_container(&supervisor, self.config.stop_timeout_secs) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error.into()), + } + match self + .client + .remove_volume(&crate::isolation::channel_volume_name(sandbox_id)) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + // The workload still owns the volume until its removal below. + Err(error) => debug!(%error, "Channel volume is still attached to workload"), + } + let Some(container_id) = self.find_container_id(sandbox_id).await? else { debug!(sandbox_id = %sandbox_id, "Sandbox container not found (already deleted)"); let vol = container::volume_name(sandbox_id); @@ -1252,6 +1501,13 @@ impl PodmanComputeDriver { }; // Remove workspace volume. + if let Err(error) = self + .client + .remove_volume(&crate::isolation::channel_volume_name(sandbox_id)) + .await + { + warn!(%sandbox_id, %error, "Failed to remove private channel volume"); + } let vol = container::volume_name(sandbox_id); if let Err(e) = self.client.remove_volume(&vol).await { warn!( @@ -1278,7 +1534,11 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; Ok(!entries.is_empty()) @@ -1292,16 +1552,18 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; let Some(entry) = entries.first() else { return Ok(None); }; if entry.state == "running" { - Ok(self - .client - .inspect_container(&entry.id) + Ok(watcher::inspect_workload(&self.client, &entry.id) .await .ok() .and_then(|inspect| driver_sandbox_from_inspect(&inspect)) @@ -1318,7 +1580,7 @@ impl PodmanComputeDriver { pub async fn list_sandboxes(&self) -> Result, ComputeDriverError> { let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER]) + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) .await .map_err(ComputeDriverError::from)?; @@ -1326,7 +1588,7 @@ impl PodmanComputeDriver { for entry in &entries { if entry.state == "running" { // Running containers need inspect for health check status. - match self.client.inspect_container(&entry.id).await { + match watcher::inspect_workload(&self.client, &entry.id).await { Ok(inspect) => { if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { sandboxes.push(sandbox); @@ -1419,7 +1681,7 @@ fn validate_apparmor_support( Ok(()) } -fn supervisor_image_pull_policy(image: &str) -> &'static str { +fn runtime_image_pull_policy(image: &str) -> &'static str { if supervisor_image_should_refresh(image) { "newer" } else { @@ -1491,34 +1753,37 @@ fn validate_rootless_local_callback_helper( ))) } -// ── Supervisor binary extraction (userns fallback) ───────────────────── +// ── Sandbox binary extraction (userns fallback) ──────────────────────── -async fn extract_supervisor_bin( +async fn extract_sandbox_bin( client: &PodmanClient, config: &PodmanComputeConfig, ) -> Result { let mut inspect = client - .inspect_image(&config.supervisor_image) + .inspect_image(&config.sandbox_runtime_image) .await .map_err(ComputeDriverError::from)?; - if supervisor_image_should_refresh(&config.supervisor_image) { + if supervisor_image_should_refresh(&config.sandbox_runtime_image) { info!( - image = %config.supervisor_image, - "Refreshing mutable podman supervisor image" + image = %config.sandbox_runtime_image, + "Refreshing mutable Podman sandbox runtime image" ); - match client.pull_image(&config.supervisor_image, "always").await { + match client + .pull_image(&config.sandbox_runtime_image, "always") + .await + { Ok(()) => { inspect = client - .inspect_image(&config.supervisor_image) + .inspect_image(&config.sandbox_runtime_image) .await .map_err(ComputeDriverError::from)?; } Err(err) => { warn!( - image = %config.supervisor_image, + image = %config.sandbox_runtime_image, error = %err, - "Failed to refresh mutable podman supervisor image; \ + "Failed to refresh mutable Podman sandbox runtime image; \ falling back to local image if present", ); } @@ -1527,36 +1792,38 @@ async fn extract_supervisor_bin( let digest = if inspect.id.is_empty() { return Err(ComputeDriverError::Precondition(format!( - "supervisor image '{}' has no ID", - config.supervisor_image, + "sandbox runtime image '{}' has no ID", + config.sandbox_runtime_image, ))); } else { &inspect.id }; - let cache_path = - openshell_core::driver_utils::supervisor_cache_path("podman-supervisor", digest) - .map_err(ComputeDriverError::Precondition)?; + let cache_path = openshell_core::driver_utils::supervisor_cache_path("podman-sandbox", digest) + .map_err(ComputeDriverError::Precondition)?; + // Unit tests use ordered Podman API stubs and intentionally exercise the + // extraction path on every create. Production reuses the immutable cache. + #[cfg(not(test))] if cache_path.is_file() { validate_linux_elf_binary(&cache_path).map_err(ComputeDriverError::Precondition)?; info!( cache_path = %cache_path.display(), - "Using cached supervisor binary" + "Using cached sandbox binary" ); return Ok(cache_path); } info!( - image = %config.supervisor_image, + image = %config.sandbox_runtime_image, cache_path = %cache_path.display(), - "Extracting supervisor binary from image" + "Extracting sandbox binary from image" ); let container_name = temp_extract_container_name(); let spec = serde_json::json!({ - "image": config.supervisor_image, + "image": config.sandbox_runtime_image, "name": container_name, - "entrypoint": [SUPERVISOR_IMAGE_BINARY_PATH], + "entrypoint": [SANDBOX_RUNTIME_IMAGE_BINARY_PATH], "command": [], }); client @@ -1570,7 +1837,7 @@ async fn extract_supervisor_bin( warn!( container = container_name, error = %err, - "Failed to remove supervisor extractor container" + "Failed to remove sandbox runtime extractor container" ); } @@ -1583,13 +1850,13 @@ async fn extract_binary_from_container( cache_path: &Path, ) -> Result { let tar_bytes = client - .copy_from_container(container_name, SUPERVISOR_IMAGE_BINARY_PATH) + .copy_from_container(container_name, SANDBOX_RUNTIME_IMAGE_BINARY_PATH) .await .map_err(ComputeDriverError::from)?; let binary_bytes = extract_first_tar_entry(&tar_bytes).map_err(|err| { ComputeDriverError::Precondition(format!( - "failed to extract supervisor binary from tar: {err}" + "failed to extract sandbox binary from tar: {err}" )) })?; @@ -1599,16 +1866,32 @@ async fn extract_binary_from_container( Ok(cache_path.to_path_buf()) } -fn userns_needs_extraction(userns: Option<&str>) -> bool { - userns.is_some_and(|mode| { - let base = mode.split(':').next().unwrap_or(mode); - !base.eq_ignore_ascii_case("host") - }) +fn podman_child_environment( + sandbox: &DriverSandbox, + image_env: &[String], +) -> HashMap { + let mut environment = image_env + .iter() + .filter_map(|entry| { + entry + .split_once('=') + .map(|(key, value)| (key.to_string(), value.to_string())) + }) + .collect::>(); + if let Some(spec) = sandbox.spec.as_ref() { + if let Some(template) = spec.template.as_ref() { + environment.extend(template.environment.clone()); + } + environment.extend(spec.environment.clone()); + } + environment.retain(|key, _| !key.starts_with("OPENSHELL_")); + environment } /// Returns `true` when userns remaps all UIDs, making host-owned bind mounts /// unreadable from inside the container. `auto` and `no-map` remap every UID; /// `keep-id` preserves the host user's UID; `host` uses the host namespace. +#[cfg(test)] fn userns_remaps_uids(userns: Option<&str>) -> bool { userns.is_some_and(|mode| { let base = mode.split(':').next().unwrap_or(mode); @@ -1621,6 +1904,10 @@ mod tests { use super::*; use crate::test_utils::{StubResponse, spawn_podman_stub}; use hyper::StatusCode; + use openshell_core::jwt::{ + CredentialEpoch, SandboxLaunchAuthentication, SecretJwt, SessionVerificationKey, + SupervisorAuthBundle, + }; use openshell_core::proto::compute::v1::{ DriverSandboxSpec, DriverSandboxTemplate, ResourceRequirements, }; @@ -1628,6 +1915,34 @@ mod tests { use std::fs; use std::path::{Path, PathBuf}; + fn launch_authentication() -> SandboxLaunchAuthentication { + SandboxLaunchAuthentication { + supervisor: SupervisorAuthBundle { + session_id: openshell_core::SandboxSessionId::new(), + runtime_generation: openshell_core::sandbox_generation::SandboxGenerationId::parse( + "generation-1", + ) + .unwrap(), + session_rotation: openshell_core::jwt::SessionRotation::new(1).unwrap(), + predecessor_session_id: None, + gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), + gateway_expires_at: i64::MAX, + sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), + sandbox_expires_at: i64::MAX, + credential_epoch: CredentialEpoch::new(1).unwrap(), + }, + gateway_id: "gateway-test".to_string(), + verification_keys: vec![SessionVerificationKey { + key_id: "test-key".to_string(), + public_key_pem: b"public-key".to_vec(), + }], + } + } + + fn encoded_launch_authentication() -> Vec { + serde_json::to_vec(&launch_authentication()).unwrap() + } + // ── socket resolution ─────────────────────────────────────────────── // // These test resolve_socket_path directly with an injected detector, so @@ -1705,6 +2020,7 @@ mod tests { "lifecycle-stop", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1720,7 +2036,7 @@ mod tests { assert_eq!( stop_requests .lock() - .expect("request log lock should not be poisoned")[1], + .expect("request log lock should not be poisoned")[2], format!( "POST {}", api_path("/libpod/containers/ctr-1/stop?timeout=10") @@ -1729,7 +2045,7 @@ mod tests { assert_eq!( stop_requests .lock() - .expect("request log lock should not be poisoned")[2], + .expect("request log lock should not be poisoned")[3], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); @@ -1741,14 +2057,38 @@ mod tests { StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, ), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + ].into_iter().chain(restart_responses()).collect(), ); + let authentication = encoded_launch_authentication(); test_driver(start_socket.clone()) - .start_sandbox("sandbox-1") + .start_sandbox("sandbox-1", "generation-1", &authentication) .await .expect("start should succeed"); start_handle.await.expect("start stub should finish"); + let restart_requests = start_requests.lock().unwrap().clone(); + assert_eq!( + restart_requests + .iter() + .filter(|request| request.starts_with("PUT ")) + .cloned() + .collect::>(), + vec![ + format!( + "PUT {}", + api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel") + ), + format!( + "PUT {}", + api_path("/libpod/containers/openshell-supervisor-sandbox-1/archive?path=%2F") + ), + ] + ); + assert!( + !restart_requests + .iter() + .any(|request| request.contains("/volumes/create") + || request.contains("/containers/create")) + ); assert_eq!( start_requests .lock() @@ -1759,7 +2099,10 @@ mod tests { start_requests .lock() .expect("request log lock should not be poisoned")[2], - format!("POST {}", api_path("/libpod/containers/ctr-1/start")) + format!( + "POST {}", + api_path("/libpod/containers/openshell-supervisor-sandbox-1/stop?timeout=10") + ) ); let _ = fs::remove_file(stop_socket); @@ -1772,6 +2115,7 @@ mod tests { "lifecycle-stop-wait", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1793,12 +2137,12 @@ mod tests { let requests = requests .lock() .expect("request log lock should not be poisoned"); - assert_eq!(requests.len(), 4); + assert_eq!(requests.len(), 5); assert_eq!( - requests[2], + requests[3], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); - assert_eq!(requests[3], requests[2]); + assert_eq!(requests[4], requests[3]); let _ = fs::remove_file(socket); } @@ -1809,6 +2153,7 @@ mod tests { "lifecycle-stop-retry", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"stopping"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new( StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, @@ -1825,9 +2170,9 @@ mod tests { let requests = requests .lock() .expect("request log lock should not be poisoned"); - assert_eq!(requests.len(), 2); + assert_eq!(requests.len(), 3); assert_eq!( - requests[1], + requests[2], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); @@ -1845,6 +2190,7 @@ mod tests { "trace-stop", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1891,19 +2237,12 @@ mod tests { use tracing_subscriber::layer::SubscriberExt as _; let _tracing_lock = openshell_otel_test_support::tracing_test_lock().await; - let (socket_path, _requests, handle) = spawn_podman_stub( + let (socket_path, requests, handle) = spawn_podman_stub( "trace-create", - vec![ - StubResponse::new(StatusCode::OK, "{}"), - StubResponse::new(StatusCode::OK, "{}"), - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), - StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + create_setup_responses(false) + .into_iter() + .chain(create_launch_responses()) + .collect(), ); let exporter = InMemorySpanExporterBuilder::new().build(); let provider = SdkTracerProvider::builder() @@ -1912,12 +2251,33 @@ mod tests { let subscriber = tracing_subscriber::registry().with(crate::otel_tracing::TRACING.layer(&provider)); + let mut sandbox = plain_sandbox("sandbox-trace", "demo"); + sandbox.spec = Some(DriverSandboxSpec { + launch_authentication: encoded_launch_authentication(), + ..DriverSandboxSpec::default() + }); test_driver(socket_path.clone()) - .create_sandbox(&plain_sandbox("sandbox-trace", "demo")) + .create_sandbox(&sandbox) .with_subscriber(subscriber) .await .expect("create should succeed"); handle.await.expect("stub should finish"); + let uploads: Vec<_> = requests + .lock() + .unwrap() + .iter() + .filter(|request| request.starts_with("PUT ")) + .cloned() + .collect(); + assert_eq!( + uploads, + [ + "/libpod/containers/workload/archive?path=%2F.openshell%2Fchannel", + "/libpod/containers/workload/archive?path=%2Fsandbox", + "/libpod/containers/supervisor/archive?path=%2F", + ] + .map(|path| format!("PUT {}", api_path(path))) + ); provider.force_flush().unwrap(); let spans = exporter.get_finished_spans().unwrap(); @@ -1929,7 +2289,6 @@ mod tests { "podman.prepare_images", "podman.prepare_storage", "podman.prepare_container", - "podman.start_container", ] { let child = spans .iter() @@ -2009,11 +2368,11 @@ mod tests { StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, ), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + ].into_iter().chain(restart_responses()).collect(), ); + let authentication = encoded_launch_authentication(); test_driver(start_socket.clone()) - .start_sandbox("sandbox-1") + .start_sandbox("sandbox-1", "generation-1", &authentication) .with_subscriber(subscriber) .await .expect("start should succeed"); @@ -2022,6 +2381,8 @@ mod tests { let (delete_socket, _requests, delete_handle) = spawn_podman_stub( "trace-delete", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached StubResponse::new(StatusCode::OK, "[]"), StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -2664,25 +3025,25 @@ mod tests { #[test] fn supervisor_pull_policy_refreshes_mutable_tags_only() { assert_eq!( - supervisor_image_pull_policy("ghcr.io/nvidia/openshell/supervisor:dev"), + runtime_image_pull_policy("ghcr.io/nvidia/openshell/supervisor:dev"), "newer" ); assert_eq!( - supervisor_image_pull_policy("ghcr.io/nvidia/openshell/supervisor:latest"), + runtime_image_pull_policy("ghcr.io/nvidia/openshell/supervisor:latest"), "newer" ); assert_eq!( - supervisor_image_pull_policy("ghcr.io/nvidia/openshell/supervisor"), + runtime_image_pull_policy("ghcr.io/nvidia/openshell/supervisor"), "newer" ); assert_eq!( - supervisor_image_pull_policy( + runtime_image_pull_policy( "ghcr.io/nvidia/openshell/supervisor:0.0.47-dev.13-g57b71c68f" ), "missing" ); assert_eq!( - supervisor_image_pull_policy("ghcr.io/nvidia/openshell/supervisor@sha256:abc123"), + runtime_image_pull_policy("ghcr.io/nvidia/openshell/supervisor@sha256:abc123"), "missing" ); } @@ -2892,6 +3253,8 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-not-found", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), // remove_volume @@ -2911,9 +3274,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) @@ -2946,11 +3309,59 @@ mod tests { name: name.to_string(), namespace: String::new(), workspace: String::new(), - spec: None, + spec: Some(DriverSandboxSpec { + launch_authentication: encoded_launch_authentication(), + ..Default::default() + }), status: None, } } + #[test] + fn child_environment_preserves_precedence_and_strips_control_keys() { + let mut sandbox = plain_sandbox("sandbox", "agent"); + sandbox.spec = Some(DriverSandboxSpec { + template: Some(DriverSandboxTemplate { + environment: HashMap::from([ + ("TEMPLATE_ONLY".to_string(), "template".to_string()), + ("OVERRIDE".to_string(), "template".to_string()), + ]), + ..Default::default() + }), + environment: HashMap::from([ + ("REQUEST_ONLY".to_string(), "request".to_string()), + ("OVERRIDE".to_string(), "request".to_string()), + ("OPENSHELL_SANDBOX_TOKEN".to_string(), "spoofed".to_string()), + ]), + ..Default::default() + }); + let image_env = vec![ + "IMAGE_ONLY=image".to_string(), + "OVERRIDE=image".to_string(), + "OPENSHELL_ENDPOINT=spoofed".to_string(), + ]; + + let environment = podman_child_environment(&sandbox, &image_env); + + assert_eq!( + environment.get("IMAGE_ONLY").map(String::as_str), + Some("image") + ); + assert_eq!( + environment.get("TEMPLATE_ONLY").map(String::as_str), + Some("template") + ); + assert_eq!( + environment.get("REQUEST_ONLY").map(String::as_str), + Some("request") + ); + assert_eq!( + environment.get("OVERRIDE").map(String::as_str), + Some("request") + ); + assert!(!environment.keys().any(|key| key.starts_with("OPENSHELL_"))); + } + fn secret_delete_request(sandbox_id: &str) -> String { format!( "DELETE {}", @@ -2961,6 +3372,209 @@ mod tests { ) } + fn restart_responses() -> Vec { + let mut archive = tar::Builder::new(Vec::new()); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let bundle = serde_json::to_vec(&crate::isolation::RestartMetadata { + generation: "generation-1".to_string(), + workload_identity: identity, + child_env: HashMap::new(), + }) + .unwrap(); + let mut header = tar::Header::new_gnu(); + header.set_size(bundle.len() as u64); + header.set_mode(0o600); + header.set_cksum(); + archive + .append_data(&mut header, "restart-metadata.json", bundle.as_slice()) + .unwrap(); + vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor stop + StubResponse::new( + StatusCode::OK, + r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"exited","Running":false},"Config":{}}"#, + ), + StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), + StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), + StubResponse::new(StatusCode::OK, ""), // refreshed supervisor auth and runtime descriptor + fence_response(), + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start + ] + } + + fn fence_response() -> StubResponse { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct HostConfig { + network_mode: &'static str, + privileged: bool, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Networks { + networks: std::collections::BTreeMap, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Fence { + host_config: HostConfig, + network_settings: Networks, + } + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Fence { + host_config: HostConfig { + network_mode: "none", + privileged: false, + }, + network_settings: Networks { + networks: std::collections::BTreeMap::default(), + }, + }) + .unwrap(), + ) + } + + fn created_response(id: &'static str) -> StubResponse { + #[derive(serde::Serialize)] + struct Created { + #[serde(rename = "Id")] + id: &'static str, + } + StubResponse::new( + StatusCode::CREATED, + serde_json::to_vec(&Created { id }).unwrap(), + ) + } + + fn image_response(id: &'static str) -> StubResponse { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Config { + user: &'static str, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Image { + id: &'static str, + config: Config, + } + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Image { + id, + config: Config { user: "1234:1235" }, + }) + .unwrap(), + ) + } + + fn sandbox_binary_archive_response() -> StubResponse { + let mut archive = Vec::new(); + { + let mut builder = tar::Builder::new(&mut archive); + let payload = b"\x7fELF-test-sandbox"; + let mut header = tar::Header::new_gnu(); + header.set_path("openshell-sandbox").unwrap(); + header.set_size(payload.len() as u64); + header.set_mode(0o755); + header.set_cksum(); + builder.append(&header, payload.as_slice()).unwrap(); + builder.finish().unwrap(); + } + StubResponse::new(StatusCode::OK, archive) + } + + fn create_setup_responses(proxy_secret: bool) -> Vec { + let mut responses = vec![ + StubResponse::new(StatusCode::OK, "{}"), // sandbox runtime pull + StubResponse::new(StatusCode::OK, "{}"), // supervisor pull + StubResponse::new(StatusCode::OK, "{}"), // workload pull + image_response("sha256:sandbox"), + created_response("identity-reader"), + StubResponse::new(StatusCode::NOT_FOUND, ""), // reserved hierarchy absent + StubResponse::new(StatusCode::NOT_FOUND, ""), // optional passwd + StubResponse::new(StatusCode::NOT_FOUND, ""), // optional group + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove stopped reader + image_response("sha256:sandbox-runtime"), + image_response("sha256:supervisor"), + StubResponse::new(StatusCode::CREATED, "{}"), // workspace volume + ]; + if proxy_secret { + responses.push(StubResponse::new(StatusCode::CREATED, "{}")); + } + responses.extend([ + image_response("sha256:sandbox-runtime"), + created_response("sandbox-runtime-extractor"), + sandbox_binary_archive_response(), + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove extractor + ]); + responses.push(StubResponse::new(StatusCode::CREATED, "{}")); // channel volume + responses + } + + fn create_launch_responses() -> Vec { + vec![ + created_response("workload"), + fence_response(), + StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), + StubResponse::new(StatusCode::OK, "").with_archive_members(&["."]), + created_response("supervisor"), + StubResponse::new(StatusCode::OK, ""), // supervisor archive + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start + ] + } + + fn channel_archive_members() -> &'static [&'static str] { + &[ + ".", + "sandbox", + "sandbox/bootstrap.json", + "sandbox/server.crt", + "sandbox/server.key", + ] + } + + #[tokio::test] + async fn reserved_image_control_root_fails_before_workload_or_secrets() { + let (path, requests, handle) = spawn_podman_stub( + "reserved-control-root", + vec![ + StubResponse::new(StatusCode::OK, "{}"), + StubResponse::new(StatusCode::OK, "{}"), + StubResponse::new(StatusCode::OK, "{}"), + image_response("sha256:image"), + created_response("identity-reader"), + StubResponse::new(StatusCode::OK, "existing reserved path"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let error = test_driver(path.clone()) + .create_sandbox(&plain_sandbox("id", "name")) + .await + .unwrap_err(); + assert!(error.to_string().contains("reserved /.openshell")); + handle.await.unwrap(); + assert!( + !requests + .lock() + .unwrap() + .iter() + .any(|request| request.contains("/libpod/volumes") + || request.contains("/libpod/secrets")) + ); + let _ = fs::remove_file(path); + } + #[tokio::test] async fn create_sandbox_removes_proxy_auth_secret_on_container_create_failure() { // A credential secret is staged before the container is created, so a @@ -2969,24 +3583,25 @@ mod tests { let auth_file = write_proxy_auth_file("create-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-container-fail", - vec![ - StubResponse::new(StatusCode::OK, "{}"), // pull supervisor image - StubResponse::new(StatusCode::OK, "{}"), // pull sandbox image - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), // inspect sandbox image - StubResponse::new(StatusCode::CREATED, "{}"), // create volume - StubResponse::new(StatusCode::CREATED, "{}"), // create proxy-auth secret - StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, r#"{"message":"boom"}"#), // create container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove volume - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove proxy-auth secret - ], + create_setup_responses(true) + .into_iter() + .chain([ + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "create failed"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace + StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret + ]) + .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); + let mut sandbox = plain_sandbox(sandbox_id, "demo"); + sandbox.spec = Some(DriverSandboxSpec { + launch_authentication: encoded_launch_authentication(), + ..DriverSandboxSpec::default() + }); driver - .create_sandbox(&plain_sandbox(sandbox_id, "demo")) + .create_sandbox(&sandbox) .await .expect_err("container create should fail"); @@ -3011,26 +3626,28 @@ mod tests { let auth_file = write_proxy_auth_file("start-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-start-fail", - vec![ - StubResponse::new(StatusCode::OK, "{}"), // pull supervisor image - StubResponse::new(StatusCode::OK, "{}"), // pull sandbox image - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), // inspect sandbox image - StubResponse::new(StatusCode::CREATED, "{}"), // create volume - StubResponse::new(StatusCode::CREATED, "{}"), // create proxy-auth secret - StubResponse::new(StatusCode::CREATED, "{}"), // create container - StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, r#"{"message":"boom"}"#), // start container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove volume - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove proxy-auth secret - ], + create_setup_responses(true) + .into_iter() + .chain(create_launch_responses().into_iter().take(7)) + .chain([ + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "supervisor start failed"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace + StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret + ]) + .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); + let mut sandbox = plain_sandbox(sandbox_id, "demo"); + sandbox.spec = Some(DriverSandboxSpec { + launch_authentication: encoded_launch_authentication(), + ..DriverSandboxSpec::default() + }); driver - .create_sandbox(&plain_sandbox(sandbox_id, "demo")) + .create_sandbox(&sandbox) .await .expect_err("container start should fail"); @@ -3055,7 +3672,9 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-proxy-auth", vec![ - StubResponse::new(StatusCode::OK, "[]"), // list_containers (not found) + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached + StubResponse::new(StatusCode::OK, "[]"), // list_containers (not found) StubResponse::new(StatusCode::NO_CONTENT, ""), // remove volume StubResponse::new(StatusCode::NO_CONTENT, ""), // remove token secret StubResponse::new(StatusCode::NO_CONTENT, ""), // remove proxy-auth secret @@ -3098,10 +3717,14 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-label-lookup", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached // list_containers by label StubResponse::new(StatusCode::OK, list_body), // single timed remove_container operation StubResponse::new(StatusCode::NO_CONTENT, ""), + // channel volume, now detached + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove_volume StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -3119,9 +3742,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!( @@ -3130,7 +3753,7 @@ mod tests { ) ); assert_eq!( - requests[2], + requests[5], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) @@ -3139,19 +3762,6 @@ mod tests { let _ = fs::remove_file(socket_path); } - #[test] - fn userns_needs_extraction_cases() { - assert!(!userns_needs_extraction(None)); - assert!(!userns_needs_extraction(Some("host"))); - assert!(!userns_needs_extraction(Some("Host"))); - assert!(userns_needs_extraction(Some("auto"))); - assert!(userns_needs_extraction(Some("auto:size=65536"))); - assert!(userns_needs_extraction(Some("keep-id"))); - assert!(userns_needs_extraction(Some("keep-id:uid=1000"))); - assert!(userns_needs_extraction(Some("no-map"))); - assert!(userns_needs_extraction(Some("private"))); - } - #[test] fn userns_remaps_uids_cases() { assert!(!userns_remaps_uids(None)); diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index fedeea3068..b8605d9f49 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -162,8 +162,7 @@ impl ComputeDriver for ComputeDriverService { .into_inner() .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.driver - .create_sandbox(&sandbox) + Box::pin(self.driver.create_sandbox(&sandbox)) .await .map_err(Status::from)?; Ok(Response::new(CreateSandboxResponse {})) @@ -201,7 +200,11 @@ impl ComputeDriver for ComputeDriverService { return Err(Status::invalid_argument("sandbox_id is required")); } self.driver - .start_sandbox(&request.sandbox_id) + .start_sandbox( + &request.sandbox_id, + &request.generation_id, + &request.launch_authentication, + ) .await .map_err(Status::from)?; Ok(Response::new(StartSandboxResponse {})) @@ -680,6 +683,8 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "forward-id", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), // remove_volume @@ -708,9 +713,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs new file mode 100644 index 0000000000..45009fb236 --- /dev/null +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -0,0 +1,463 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman-owned provisioning for the common authenticated isolation channel. + +use std::collections::{BTreeMap, HashMap}; +#[cfg(test)] +use std::io::Read; +use std::path::PathBuf; + +use openshell_core::ComputeDriverError; +use openshell_core::proto::compute::v1::DriverSandbox; +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; +use openshell_sandbox_backend::boundary_protocol::{ + BoundaryConfig, BoundaryListener, GatewayVerificationKey, SandboxRuntimeDescriptor, + SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, + generate_sandbox_tls_material, +}; +use serde::{Deserialize, Serialize}; + +pub const LABEL_ROLE: &str = "openshell.io/isolation-role"; +pub const WORKLOAD_FILTER: &str = "openshell.io/isolation-role=sandbox"; +pub const CHANNEL_ROOT: &str = "/.openshell/channel"; +pub const BOOTSTRAP_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; +pub const RUNTIME_DESCRIPTOR_PATH: &str = "/.openshell/supervisor/runtime-descriptor.json"; +pub const AUTH_BUNDLE_PATH: &str = "/.openshell/supervisor/auth.json"; +pub const RESTART_METADATA_PATH: &str = "/.openshell/supervisor/restart-metadata.json"; +const SOCKET_PATH: &str = "/.openshell/channel/sandbox/control.sock"; + +pub fn supervisor_name(id: &str) -> String { + format!("openshell-supervisor-{id}") +} +pub fn channel_volume_name(id: &str) -> String { + format!("openshell-channel-{id}") +} + +fn invalid(error: impl std::fmt::Display) -> ComputeDriverError { + ComputeDriverError::Precondition(error.to_string()) +} + +/// Resolve policy names against the pinned workload image, never the gateway. +pub fn resolve_identity( + sandbox: &DriverSandbox, + image_id: &str, + image_user: &str, + passwd: &[u8], + group: &[u8], +) -> Result { + let passwd = std::str::from_utf8(passwd).map_err(invalid)?; + let group = std::str::from_utf8(group).map_err(invalid)?; + let accounts: Vec<_> = passwd + .lines() + .filter_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + fields.next()?; + Some(( + name, + fields.next()?.parse::().ok()?, + fields.next()?.parse::().ok()?, + )) + }) + .collect(); + let groups: Vec<_> = group + .lines() + .filter_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + fields.next()?; + Some((name, fields.next()?.parse::().ok()?, fields.next()?)) + }) + .collect(); + let request = sandbox + .spec + .as_ref() + .and_then(|spec| spec.workload_identity.as_ref()); + let requested_user = request.map_or("", |identity| identity.user.trim()); + let requested_group = request.map_or("", |identity| identity.group.trim()); + let (image_user, image_group) = image_user.split_once(':').unwrap_or((image_user, "")); + let user = if requested_user.is_empty() { + image_user + } else { + requested_user + }; + let group = if requested_group.is_empty() { + image_group + } else { + requested_group + }; + let account = accounts + .iter() + .find(|(name, uid, _)| *name == user || user.parse::().ok() == Some(*uid)); + let uid = user + .parse() + .ok() + .or_else(|| account.map(|(_, uid, _)| *uid)) + .ok_or_else(|| invalid("configure a non-root workload user present in the pinned image"))?; + let gid = if group.is_empty() { + account.map(|(_, _, gid)| *gid) + } else { + group.parse().ok().or_else(|| { + groups + .iter() + .find(|(name, _, _)| *name == group) + .map(|(_, gid, _)| *gid) + }) + } + .ok_or_else(|| { + invalid("configure an explicit workload group for a UID without an image passwd entry") + })?; + let supplemental = account.map_or_else(Vec::new, |(username, _, _)| { + groups + .iter() + .filter(|(_, id, members)| { + *id != gid && members.split(',').any(|member| member == *username) + }) + .map(|(_, gid, _)| *gid) + .collect() + }); + let source = if requested_user.is_empty() && requested_group.is_empty() { + "image" + } else { + "policy" + }; + ResolvedWorkloadIdentity::new(uid, gid, supplemental, source.into(), image_id.into()) + .map_err(invalid) +} + +pub struct BootstrapArchives { + pub channel: Vec, + pub workspace: Vec, + pub supervisor: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestartMetadata { + pub(crate) generation: String, + pub(crate) workload_identity: ResolvedWorkloadIdentity, + pub(crate) child_env: HashMap, +} + +/// The shared volume contains only sandbox credentials. Supervisor credentials, +/// gateway authorization, and the restart copy never enter that volume. +pub fn bootstrap_archives( + sandbox_id: &str, + container_id: &str, + generation: &str, + identity: &ResolvedWorkloadIdentity, + child_env: HashMap, + launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, +) -> Result { + launch_authentication.validate().map_err(invalid)?; + let session_id = launch_authentication.supervisor.session_id; + let tls = generate_sandbox_tls_material(session_id).map_err(invalid)?; + let resource_claims = BTreeMap::from([ + ("podman.container_id".into(), container_id.into()), + ( + "podman.image_identity".into(), + identity.resource_digest.clone(), + ), + ]); + let driver_fence = DriverFenceEvidence::Podman { + container_id: container_id.into(), + network_mode: "none".into(), + unexpected_networks: Vec::new(), + }; + let runtime_generation = launch_authentication + .supervisor + .runtime_generation + .to_string(); + let verification_keys = launch_authentication + .verification_keys + .iter() + .map(|key| { + String::from_utf8(key.public_key_pem.clone()) + .map(|public_key_pem| GatewayVerificationKey { + key_id: key.key_id.clone(), + public_key_pem, + }) + .map_err(invalid) + }) + .collect::, _>>()?; + let config = BoundaryConfig { + boundary_id: sandbox_id.into(), + generation: runtime_generation.clone(), + session_id, + session_rotation: launch_authentication.supervisor.session_rotation, + gateway_id: launch_authentication.gateway_id.clone(), + verification_keys, + listener: BoundaryListener::Unix { + socket_path: PathBuf::from(SOCKET_PATH), + tls: SandboxTlsServerConfig { + certificate_chain_path: PathBuf::from("/.openshell/channel/sandbox/server.crt"), + private_key_path: PathBuf::from("/.openshell/channel/sandbox/server.key"), + }, + }, + resource_claims: resource_claims.clone(), + resource_claim_files: BTreeMap::new(), + workload_identity: identity.clone(), + driver_fence: driver_fence.clone(), + child_env: child_env.clone(), + }; + let runtime_descriptor = SandboxRuntimeDescriptor { + boundary_id: sandbox_id.into(), + generation: runtime_generation, + session_id, + transport: SandboxTransport::Unix { + socket_path: PathBuf::from(SOCKET_PATH), + }, + tls: SandboxTlsClientConfig { + server_name: tls.server_name, + trust_anchor_pem: tls.trust_anchor_pem, + }, + host_gateway_ip: None, + resource_claims, + workload_identity: identity.clone(), + driver_fence, + }; + // Libpod resolves the requested upload destination once for a stopped + // container. Archive entries must be relative to the selected named volume, + // not rootfs paths that the volume would shadow on container start. + let mut channel = Archive::new(identity); + channel.directory(".", 0o755, false)?; + channel.directory("sandbox", 0o711, true)?; + channel.file( + "sandbox/bootstrap.json", + &serde_json::to_vec(&config).map_err(invalid)?, + )?; + channel.file("sandbox/server.crt", tls.certificate_chain_pem.as_bytes())?; + channel.file("sandbox/server.key", tls.private_key_pem.as_bytes())?; + let channel = channel.finish()?; + let mut workspace = Archive::new(identity); + workspace.directory(".", 0o700, true)?; + let mut supervisor = Archive::new(identity); + supervisor.directory(".openshell", 0o755, false)?; + supervisor.directory(".openshell/supervisor", 0o700, true)?; + supervisor.file( + RUNTIME_DESCRIPTOR_PATH, + &serde_json::to_vec(&runtime_descriptor).map_err(invalid)?, + )?; + supervisor.file( + AUTH_BUNDLE_PATH, + &serde_json::to_vec(&launch_authentication.supervisor).map_err(invalid)?, + )?; + let restart_metadata = RestartMetadata { + generation: generation.to_string(), + workload_identity: identity.clone(), + child_env, + }; + supervisor.file( + RESTART_METADATA_PATH, + &serde_json::to_vec(&restart_metadata).map_err(invalid)?, + )?; + Ok(BootstrapArchives { + channel, + workspace: workspace.finish()?, + supervisor: supervisor.finish()?, + }) +} + +pub fn restart_metadata_from_slice(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(invalid) +} + +struct Archive<'a> { + builder: tar::Builder>, + identity: &'a ResolvedWorkloadIdentity, +} +impl<'a> Archive<'a> { + fn new(identity: &'a ResolvedWorkloadIdentity) -> Self { + Self { + builder: tar::Builder::new(Vec::new()), + identity, + } + } + fn directory(&mut self, path: &str, mode: u32, owned: bool) -> Result<(), ComputeDriverError> { + self.append(path, mode, owned, tar::EntryType::Directory, &[]) + } + fn file(&mut self, path: &str, content: &[u8]) -> Result<(), ComputeDriverError> { + self.append( + path.trim_start_matches('/'), + 0o600, + true, + tar::EntryType::Regular, + content, + ) + } + fn append( + &mut self, + path: &str, + mode: u32, + owned: bool, + kind: tar::EntryType, + content: &[u8], + ) -> Result<(), ComputeDriverError> { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(kind); + header.set_mode(mode); + header.set_uid(if owned { + u64::from(self.identity.uid) + } else { + 0 + }); + header.set_gid(if owned { + u64::from(self.identity.gid) + } else { + 0 + }); + header.set_size(content.len() as u64); + header.set_mtime(0); + header.set_cksum(); + self.builder + .append_data(&mut header, path, content) + .map_err(invalid) + } + fn finish(self) -> Result, ComputeDriverError> { + self.builder.into_inner().map_err(invalid) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::jwt::{ + CredentialEpoch, SandboxLaunchAuthentication, SecretJwt, SessionVerificationKey, + SupervisorAuthBundle, + }; + + fn authentication() -> SandboxLaunchAuthentication { + SandboxLaunchAuthentication { + supervisor: SupervisorAuthBundle { + session_id: openshell_core::SandboxSessionId::new(), + runtime_generation: openshell_core::sandbox_generation::SandboxGenerationId::parse( + "generation-1", + ) + .unwrap(), + session_rotation: openshell_core::jwt::SessionRotation::new(1).unwrap(), + predecessor_session_id: None, + gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), + gateway_expires_at: i64::MAX, + sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), + sandbox_expires_at: i64::MAX, + credential_epoch: CredentialEpoch::new(1).unwrap(), + }, + gateway_id: "gateway-test".to_string(), + verification_keys: vec![SessionVerificationKey { + key_id: "test-key".to_string(), + public_key_pem: b"public-key".to_vec(), + }], + } + } + + #[test] + fn identity_uses_pinned_image_accounts_and_rejects_root() { + let sandbox = DriverSandbox::default(); + let passwd = b"root:x:0:0:root:/root:/bin/sh\nagent:x:1000:1001::/home/agent:/bin/sh\n"; + let groups = b"agent:x:1001:\ndata:x:2000:agent\n"; + let identity = + resolve_identity(&sandbox, "sha256:pinned", "agent", passwd, groups).unwrap(); + assert_eq!((identity.uid, identity.gid), (1000, 1001)); + assert_eq!(identity.supplementary_gids, vec![2000]); + assert_eq!(identity.resource_digest, "sha256:pinned"); + assert!(resolve_identity(&sandbox, "sha256:pinned", "root", passwd, groups).is_err()); + assert!(resolve_identity(&sandbox, "sha256:pinned", "", passwd, groups).is_err()); + assert!(resolve_identity(&sandbox, "sha256:pinned", "2000", passwd, groups).is_err()); + } + + fn files(bytes: &[u8]) -> BTreeMap> { + tar::Archive::new(bytes) + .entries() + .unwrap() + .filter_map(|entry| { + let mut entry = entry.unwrap(); + if !entry.header().entry_type().is_file() { + return None; + } + let path = entry.path().unwrap().into_owned(); + assert_eq!(entry.header().mode().unwrap(), 0o600); + assert_eq!(entry.header().uid().unwrap(), 1000); + let mut content = Vec::new(); + entry.read_to_end(&mut content).unwrap(); + Some((path, content)) + }) + .collect() + } + + #[test] + fn archives_separate_supervisor_credentials_and_bind_one_channel() { + let identity = ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let authentication = authentication(); + let child_env = HashMap::from([("PATH".to_string(), "/agent/bin".to_string())]); + let archives = bootstrap_archives( + "sandbox", + "container", + "generation-1", + &identity, + child_env.clone(), + &authentication, + ) + .unwrap(); + let workload = files(&archives.channel); + let supervisor = files(&archives.supervisor); + let mut workspace = tar::Archive::new(archives.workspace.as_slice()); + let mut entries = workspace.entries().unwrap(); + let root = entries.next().unwrap().unwrap(); + assert_eq!(root.path().unwrap().as_ref(), std::path::Path::new(".")); + assert!(root.header().entry_type().is_dir()); + assert_eq!(root.header().uid().unwrap(), u64::from(identity.uid)); + assert_eq!(root.header().gid().unwrap(), u64::from(identity.gid)); + assert_eq!(root.header().mode().unwrap(), 0o700); + assert!(entries.next().is_none()); + assert_eq!(workload.len(), 3); + assert_eq!(supervisor.len(), 3); + assert!(workload.keys().all(|path| path.starts_with("sandbox"))); + assert!( + supervisor + .keys() + .all(|path| path.starts_with(".openshell/supervisor")) + ); + let config: BoundaryConfig = serde_json::from_slice( + workload + .get(&PathBuf::from("sandbox/bootstrap.json")) + .unwrap(), + ) + .unwrap(); + let runtime_descriptor: SandboxRuntimeDescriptor = serde_json::from_slice( + supervisor + .get(&PathBuf::from( + RUNTIME_DESCRIPTOR_PATH.trim_start_matches('/'), + )) + .unwrap(), + ) + .unwrap(); + assert_eq!(config.boundary_id, runtime_descriptor.boundary_id); + assert_eq!(config.session_id, runtime_descriptor.session_id); + assert_eq!(config.driver_fence, runtime_descriptor.driver_fence); + assert_eq!(config.workload_identity, identity); + runtime_descriptor.driver_fence.validate().unwrap(); + let restart_metadata: RestartMetadata = serde_json::from_slice( + supervisor + .get(&PathBuf::from( + RESTART_METADATA_PATH.trim_start_matches('/'), + )) + .unwrap(), + ) + .unwrap(); + assert_eq!(restart_metadata.workload_identity, identity); + assert_eq!(restart_metadata.child_env, child_env); + let restart_bytes = serde_json::to_vec(&restart_metadata).unwrap(); + assert!( + !restart_bytes + .windows(b"PRIVATE KEY".len()) + .any(|window| window == b"PRIVATE KEY") + ); + } +} diff --git a/crates/openshell-driver-podman/src/lib.rs b/crates/openshell-driver-podman/src/lib.rs index 115e64eb2f..fa06cf4864 100644 --- a/crates/openshell-driver-podman/src/lib.rs +++ b/crates/openshell-driver-podman/src/lib.rs @@ -6,6 +6,7 @@ pub mod config; pub(crate) mod container; pub mod driver; pub mod grpc; +mod isolation; pub mod otel_tracing; mod socket_discovery; #[cfg(test)] diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index e4554602f4..62a2dde5a1 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -101,7 +101,11 @@ struct Args { )] health_check_interval_secs: Option, - /// OCI image containing the openshell-sandbox supervisor binary. + /// OCI image containing the `openshell-sandbox` runtime binary. + #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE")] + sandbox_runtime_image: Option, + + /// OCI image containing the `openshell-supervisor` control binary. #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] supervisor_image: Option, @@ -207,6 +211,9 @@ async fn main() -> Result<()> { ssh_socket_path: args.sandbox_ssh_socket_path, network_name: args.network_name, stop_timeout_secs: args.stop_timeout, + sandbox_runtime_image: args + .sandbox_runtime_image + .unwrap_or_else(openshell_core::config::default_sandbox_runtime_image), supervisor_image: args .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), diff --git a/crates/openshell-driver-podman/src/test_utils.rs b/crates/openshell-driver-podman/src/test_utils.rs index ec5c8f7f11..25cbcb4ac2 100644 --- a/crates/openshell-driver-podman/src/test_utils.rs +++ b/crates/openshell-driver-podman/src/test_utils.rs @@ -3,7 +3,7 @@ //! Shared test helpers for openshell-driver-podman unit tests. -use http_body_util::Full; +use http_body_util::{BodyExt as _, Full}; use hyper::StatusCode; use hyper::body::Bytes; use hyper::server::conn::http1; @@ -20,16 +20,18 @@ use tokio::net::UnixListener; #[derive(Clone)] pub struct StubResponse { pub status: StatusCode, - pub body: String, + pub body: Bytes, pub delay: Duration, + pub archive_members: Option>, } impl StubResponse { - pub fn new(status: StatusCode, body: impl Into) -> Self { + pub fn new(status: StatusCode, body: impl Into) -> Self { Self { status, body: body.into(), delay: Duration::ZERO, + archive_members: None, } } @@ -37,6 +39,11 @@ impl StubResponse { self.delay = delay; self } + + pub fn with_archive_members(mut self, members: &[&str]) -> Self { + self.archive_members = Some(members.iter().map(PathBuf::from).collect()); + self + } } /// Generate a unique Unix socket path for a test. @@ -88,7 +95,7 @@ pub fn spawn_podman_stub( let result = http1::Builder::new() .serve_connection( TokioIo::new(stream), - service_fn(move |req| { + service_fn(move |req: hyper::Request| { let log = log.clone(); let queue = queue.clone(); async move { @@ -104,11 +111,22 @@ pub fn spawn_podman_stub( .expect("response queue lock should not be poisoned") .pop_front() .expect("stub response should exist"); + if let Some(expected_members) = &response.archive_members { + assert_eq!(req.method(), hyper::Method::PUT); + let body = req.into_body().collect().await.unwrap().to_bytes(); + let mut archive = tar::Archive::new(body.as_ref()); + let members: Vec<_> = archive + .entries() + .unwrap() + .map(|entry| entry.unwrap().path().unwrap().into_owned()) + .collect(); + assert_eq!(&members, expected_members); + } tokio::time::sleep(response.delay).await; Ok::<_, Infallible>( hyper::Response::builder() .status(response.status) - .body(Full::new(Bytes::from(response.body))) + .body(Full::new(response.body)) .expect("stub response should build"), ) } diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index b99ecdd540..6e5eb52f27 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -139,14 +139,16 @@ pub async fn start_watch( let mut event_rx = client.events_stream(LABEL_MANAGED_FILTER).await?; // 2. List existing containers for initial state sync. - let existing = client.list_containers(&[LABEL_MANAGED_FILTER]).await?; + let existing = client + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) + .await?; for entry in &existing { // For running containers, use inspect to get full state including // health check status — matching the same condition derivation used // for live events. if entry.state == "running" { - match client.inspect_container(&entry.id).await { + match inspect_workload(&client, &entry.id).await { Ok(inspect) => { if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { if tx.send(Ok(sandbox_event(sandbox))).await.is_err() { @@ -246,11 +248,34 @@ async fn map_podman_event( return None; } + if event + .actor + .attributes + .get(crate::isolation::LABEL_ROLE) + .is_some_and(|role| role == "supervisor") + { + let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); + let workloads = client + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) + .await + .ok()?; + let workload = workloads.first()?; + return inspect_workload(client, &workload.id) + .await + .ok() + .and_then(|inspect| driver_sandbox_from_inspect(&inspect)) + .map(sandbox_event); + } + match event.action.as_str() { "remove" => Some(deleted_event(sandbox_id.clone())), "create" | "start" | "stop" | "die" | "health_status" => { // Inspect the container to get current state. - match client.inspect_container(container_id).await { + match inspect_workload(client, container_id).await { Ok(inspect) => { if lifecycle_event_fences.matches_previous_exit( event, @@ -327,6 +352,54 @@ async fn map_podman_event( } } +/// A workload is ready only when its independent supervisor is healthy. This +/// check runs both on watch reconciliation and on events, and contains a lost +/// supervisor even when the gateway missed the original exit event. +pub async fn inspect_workload( + client: &PodmanClient, + id: &str, +) -> Result { + let mut workload = client.inspect_container(id).await?; + if workload + .config + .labels + .get(crate::isolation::LABEL_ROLE) + .is_none_or(|role| role != "sandbox") + { + return Ok(workload); + } + let Some(sandbox_id) = workload.config.labels.get(LABEL_SANDBOX_ID) else { + return Ok(workload); + }; + let supervisor = client + .inspect_container(&crate::isolation::supervisor_name(sandbox_id)) + .await; + if workload.state.running { + match supervisor { + Ok(supervisor) if supervisor.state.running => { + workload.state.health = supervisor.state.health; + } + Ok(supervisor) + if supervisor.state.status == "configured" + || supervisor.state.status == "created" => + { + workload.state.health = Some(HealthState { + status: "starting".into(), + }); + } + // Both containers exist before initial start. A missing or exited + // companion therefore requires containment, including after a + // gateway restart that missed the original Podman exit event. + Ok(_) | Err(PodmanApiError::NotFound(_)) => { + client.stop_container(&workload.id, 0).await?; + workload = client.inspect_container(&workload.id).await?; + } + Err(error) => return Err(error), + } + } + Ok(workload) +} + /// Construct a `DriverSandbox` from common fields. /// /// Centralises the boilerplate that every event/inspect/list path shares: @@ -510,6 +583,65 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { mod tests { use super::*; + #[tokio::test] + async fn missing_supervisor_stops_workload_during_reconciliation() { + use crate::test_utils::{StubResponse, spawn_podman_stub}; + use hyper::StatusCode; + let (path, requests, handle) = spawn_podman_stub( + "lost-supervisor", + vec![ + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"running","Running":true},"Config":{"Labels":{"openshell.ai/sandbox-id":"test","openshell.io/isolation-role":"sandbox"}}}"#, + ), + StubResponse::new(StatusCode::NOT_FOUND, "missing companion"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"exited","Running":false},"Config":{}}"#, + ), + ], + ); + let client = PodmanClient::new(path.clone()); + let inspected = inspect_workload(&client, "workload").await.unwrap(); + assert!(!inspected.state.running); + handle.await.unwrap(); + assert!( + requests + .lock() + .unwrap() + .iter() + .any(|request| request.ends_with("/libpod/containers/workload/stop?timeout=0")) + ); + let _ = std::fs::remove_file(path); + } + + #[tokio::test] + async fn created_supervisor_keeps_bootstrapping_workload_starting() { + use crate::test_utils::{StubResponse, spawn_podman_stub}; + use hyper::StatusCode; + let (path, requests, handle) = spawn_podman_stub( + "starting-supervisor", + vec![ + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"running","Running":true},"Config":{"Labels":{"openshell.ai/sandbox-id":"test","openshell.io/isolation-role":"sandbox"}}}"#, + ), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"configured","Running":false},"Config":{}}"#, + ), + ], + ); + let client = PodmanClient::new(path.clone()); + let inspected = inspect_workload(&client, "workload").await.unwrap(); + assert!(inspected.state.running); + assert_eq!(inspected.state.health.unwrap().status, "starting"); + handle.await.unwrap(); + assert_eq!(requests.lock().unwrap().len(), 2); + let _ = std::fs::remove_file(path); + } + fn podman_event(action: &str, sandbox_id: &str, time_nano: i64) -> PodmanEvent { PodmanEvent { event_type: "container".to_string(), diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index e5b16d00d0..af2e97a94f 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -222,7 +222,8 @@ overrides that persist across package upgrades. | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | | `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | | `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | -| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | +| `[openshell.drivers.podman].sandbox_runtime_image` | `ghcr.io/nvidia/openshell/sandbox:latest` | Static musl sandbox runtime image mounted into Podman workloads. | +| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Dynamic glibc supervisor image used outside the workload. | | `[openshell.gateway].guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Gateway-owned client TLS material injected into the selected local driver and mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | @@ -270,6 +271,7 @@ To pin specific image versions instead of `:latest`, set these values in `[openshell.drivers.podman]`: ```toml +sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:v0.0.37" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:v0.0.37" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:v0.0.37" ``` diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index ed9b38383d..7d2f5bb790 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -543,6 +543,9 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/b image_pull_policy = "if_not_present" image_pull_secrets = ["regcred"] # Defaults to the gateway version; override to pin a specific build. +# sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" +sandbox_runtime_image_pull_policy = "if_not_present" +# Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" supervisor_image_pull_policy = "if_not_present" @@ -640,12 +643,12 @@ map Windows Sandboxing provider events into the gateway's OCSF stream. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["mxc"] +compute_driver = "mxc" [openshell.drivers.mxc] wxc_exec_path = "C:\\mxc-kit\\bin\\wxc-exec.exe" @@ -694,9 +697,9 @@ sandbox_label = "docker-dev" # Optional override. When omitted, the gateway derives # https://host.openshell.internal: for this driver. grpc_endpoint = "https://host.openshell.internal:17670" -# The workload runtime and supervisor companion use separate images. Both -# default to the gateway version; override either to pin a specific build. +# Workload-side runtime. Defaults to the gateway version. # sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" +# Supervisor runtime. Defaults to the gateway version. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" @@ -706,10 +709,9 @@ host_gateway_ip = "172.17.0.1" enable_bind_mounts = false # Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 -# Omit this field to keep Docker's runtime-selected AppArmor profile. -# Localhost/ requires an operator-loaded profile; Unconfined is an -# explicit operator opt-out. -# app_armor_profile = "Localhost/openshell-sandbox" +# Explicit supervisor-compatible default. RuntimeDefault requires Docker to +# report AppArmor support; Localhost/ requires an operator-loaded profile. +app_armor_profile = "Unconfined" # Corporate TLS egress proxy. These are supervisor argv settings, not workload # environment variables. Do not embed credentials in the URL. https_proxy = "https://proxy.corp.example:8443" @@ -727,7 +729,7 @@ Use `sandbox_label` for Docker configurations. The legacy ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount. Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. +Each Podman sandbox uses two containers. The workload container runs `openshell-sandbox` with `network=none`; the supervisor container runs on the host network and initiates policy-approved upstream connections. A private volume carries their authenticated Unix-domain socket. Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] @@ -760,7 +762,9 @@ network_name = "openshell" # host_gateway_ip = "192.168.127.254" ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 -# Defaults to the gateway version; override to pin a specific build. +# Statically linked workload-side runtime. Defaults to the gateway version. +# sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" +# Dynamically linked supervisor runtime. Defaults to the gateway version. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" # Unsafe operator override. Host bind mounts, including Podman local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can @@ -863,18 +867,17 @@ health_check_interval_secs = 10 # explicit container-reachable TCP endpoint, for provider token exchange. # provider_spiffe_workload_api_socket = "/run/spire/agent.sock" # provider_spiffe_workload_api_socket = "tcp:169.254.1.2:8081" -# Omit app_armor_profile to preserve Podman's runtime-selected profile. -# Set Unconfined only when the supervisor's mount setup requires it. +# Omit app_armor_profile to preserve Podman's runtime-selected workload profile. # Explicit RuntimeDefault and Localhost/ require Podman to report # AppArmor support. -# app_armor_profile = "Unconfined" +# app_armor_profile = "RuntimeDefault" ``` Use `ssh_socket_path` for Podman configurations. The legacy `sandbox_ssh_socket_path` key is rejected. When `app_armor_profile` is omitted, OpenShell sends no override and Podman applies its runtime-selected profile. -Set `Unconfined` explicitly only when the deployment requires the supervisor's -mount setup to bypass that profile. +The setting applies to the workload container; the supervisor retains Podman's +runtime-selected profile. ### MicroVM diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 67a45a3ae1..76548f60c7 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -42,11 +42,11 @@ with the exact exit code. Driver and supervisor failures remain `Error`. ## Configure a Compute Driver -Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_drivers` in the gateway TOML file: +Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_driver` in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["docker"] +compute_driver = "docker" ``` Reserved built-in values are `docker`, `podman`, `kubernetes`, `vm`, and `mxc`. @@ -54,13 +54,13 @@ The `mxc` driver is available only in native Windows gateway builds. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. +When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_COMPUTE_DRIVER=vm` in the launch environment. Common gateway options: | Gateway TOML option | Description | |---|---| -| `compute_drivers = [""]` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | +| `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. @@ -70,7 +70,7 @@ the gateway at the Unix socket the operator has already provisioned: ```toml [openshell.gateway] -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma.sock" @@ -175,7 +175,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains `/openshell-sandbox`; the supervisor companion image contains `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains `/openshell-sandbox`; the supervisor image contains `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -251,9 +251,11 @@ namespace roots. These checks do not make host bind mounts safe. The gateway talks to the Podman API socket. The Podman driver requires Podman 5.x, cgroups v2, rootless networking, and an active Podman user socket. When `socket_path` is not set, the driver probes known socket paths, then uses the `podman` CLI to resolve the active native or machine-backed connection. It fails to start if neither method finds a socket. +The agent workload uses `network=none`. Its trusted supervisor companion uses Podman's host network for gateway callbacks and policy-approved upstream connections. + For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -347,14 +349,14 @@ For maintainer-level implementation details, refer to the [VM driver README](htt The VM driver is opt-in. Release packages can install `openshell-driver-vm`, but the gateway does not select it unless you configure the driver explicitly. -Enable VM by setting `compute_drivers = ["vm"]` in the gateway TOML file: +Enable VM by setting `compute_driver = "vm"` in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["vm"] +compute_driver = "vm" ``` -For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. +For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. @@ -412,7 +414,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | Gateway configuration | Helm value | Description | |---|---|---| -| `compute_drivers = ["kubernetes"]` | Not applicable | Select the Kubernetes compute driver. | +| `compute_driver = "kubernetes"` | Not applicable | Select the Kubernetes compute driver. | | `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. | | `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | @@ -421,7 +423,9 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | -| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the trusted runtime image that provides the `openshell-sandbox` and `openshell-supervisor` binaries. The default repository with an empty tag uses the version pinned into the gateway. | +| `sandbox_runtime_image` | `sandboxRuntime.image.repository` / `sandboxRuntime.image.tag` | Override the image that provides `openshell-sandbox`. The default repository with an empty tag uses the version pinned into the gateway. | +| `sandbox_runtime_image_pull_policy` | `sandboxRuntime.image.pullPolicy` | Set the Kubernetes image pull policy for the sandbox runtime image. | +| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the image that provides `openshell-supervisor`. The default repository with an empty tag uses the version pinned into the gateway. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `sandbox_runtime.network_policy_enforced` | `supervisor.sandboxRuntime.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. | | `sandbox_runtime.boundary_port` | `supervisor.sandboxRuntime.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. | diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index 6eff3b5615..a53708c61a 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -26,5 +26,6 @@ health_check_interval_secs = 10 network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" ssh_socket_path = "/run/openshell/ssh.sock" +sandbox_runtime_image = "localhost/openshell/sandbox:e2e-vm" supervisor_image = "localhost/openshell/supervisor:e2e-vm" app_armor_profile = "Unconfined" diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 2b8124ccd6..b06f76678a 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -213,6 +213,7 @@ if external: "external_driver_proxy": False, "external_driver_app_armor": False, "external_driver_environment": { + "XDG_DATA_HOME": f"/tmp/{variant}-driver-data", "OPENSHELL_COMPUTE_DRIVER_SOCKET": driver_socket, "OPENSHELL_PODMAN_SOCKET": podman_socket, "OPENSHELL_SANDBOX_IMAGE": sandbox_runtime, diff --git a/e2e/parity/verify-results.py b/e2e/parity/verify-results.py index 8f4942dda3..4e8a4619bb 100644 --- a/e2e/parity/verify-results.py +++ b/e2e/parity/verify-results.py @@ -262,6 +262,7 @@ def verify_variant( sandbox_id = launch.get("sandbox_image_id") sandbox_digest = launch.get("sandbox_image_digest") sandbox_runtime = launch.get("sandbox_runtime_image") + sandbox_boundary_image = launch.get("sandbox_boundary_image") sandbox_match = ( DIGEST_REFERENCE_RE.fullmatch(sandbox_runtime) if isinstance(sandbox_runtime, str) @@ -285,6 +286,10 @@ def verify_variant( sandbox_request == sandbox_runtime, f"{launch_path}: sandbox image request was not the resolved digest reference", ) + require( + isinstance(sandbox_boundary_image, str) and sandbox_boundary_image, + f"{launch_path}: sandbox boundary image is missing", + ) if not external: require( podman_config["default_image"] == sandbox_runtime @@ -369,6 +374,7 @@ def verify_variant( ) driver_environment = launch.get("external_driver_environment") expected_environment_keys = { + "XDG_DATA_HOME", "OPENSHELL_COMPUTE_DRIVER_SOCKET", "OPENSHELL_PODMAN_SOCKET", "OPENSHELL_SANDBOX_IMAGE", @@ -378,6 +384,7 @@ def verify_variant( "OPENSHELL_GATEWAY_PORT", "OPENSHELL_NETWORK_NAME", "OPENSHELL_STOP_TIMEOUT", + "OPENSHELL_SANDBOX_RUNTIME_IMAGE", "OPENSHELL_SUPERVISOR_IMAGE", "OPENSHELL_PODMAN_TLS_CA", "OPENSHELL_PODMAN_TLS_CERT", @@ -389,6 +396,10 @@ def verify_variant( and set(driver_environment) == expected_environment_keys, f"{launch_path}: external driver allowlisted environment is incomplete", ) + require( + Path(driver_environment["XDG_DATA_HOME"]).is_absolute(), + f"{launch_path}: external driver data directory is not absolute", + ) require( driver_environment["OPENSHELL_COMPUTE_DRIVER_SOCKET"] == podman_config["socket_path"], @@ -402,7 +413,7 @@ def verify_variant( f"{launch_path}: external driver Podman socket is not isolated", ) require( - driver_environment["OPENSHELL_SANDBOX_IMAGE"] == sandbox_runtime + driver_environment["OPENSHELL_SANDBOX_IMAGE"] == sandbox_request and driver_environment["OPENSHELL_SANDBOX_IMAGE_PULL_POLICY"] == expected_policy and driver_environment["OPENSHELL_HEALTH_CHECK_INTERVAL_SECS"] == 10 @@ -412,6 +423,8 @@ def verify_variant( and driver_environment["OPENSHELL_NETWORK_NAME"] and isinstance(driver_environment["OPENSHELL_STOP_TIMEOUT"], int) and driver_environment["OPENSHELL_STOP_TIMEOUT"] >= 0 + and driver_environment["OPENSHELL_SANDBOX_RUNTIME_IMAGE"] + == sandbox_boundary_image and driver_environment["OPENSHELL_SUPERVISOR_IMAGE"] == runtime_image and driver_environment["OPENSHELL_ENABLE_BIND_MOUNTS"] is True, f"{launch_path}: external driver allowlisted runtime inputs differ", diff --git a/e2e/rust/tests/podman_gateway_start.rs b/e2e/rust/tests/podman_gateway_start.rs index 28a9ceb15c..7e159ffa78 100644 --- a/e2e/rust/tests/podman_gateway_start.rs +++ b/e2e/rust/tests/podman_gateway_start.rs @@ -38,10 +38,10 @@ const SANDBOX_NAME_LABEL: &str = "openshell.ai/sandbox-name"; /// harness), fall back to plain `podman`, leaving Linux behavior unchanged. fn podman_command() -> Command { let mut command = Command::new("podman"); - if let Ok(socket) = std::env::var("OPENSHELL_PODMAN_SOCKET") { - if !socket.is_empty() { - command.arg("--url").arg(format!("unix://{socket}")); - } + if let Ok(socket) = std::env::var("OPENSHELL_PODMAN_SOCKET") + && !socket.is_empty() + { + command.arg("--url").arg(format!("unix://{socket}")); } command } @@ -49,7 +49,15 @@ fn podman_command() -> Command { fn sandbox_container_running(sandbox_name: &str) -> Result { let sandbox_name_filter = format!("label={SANDBOX_NAME_LABEL}={sandbox_name}"); let output = podman_command() - .args(["ps", "-aq", "--filter", MANAGED_BY_LABEL_FILTER, "--filter"]) + .args([ + "ps", + "-aq", + "--filter", + MANAGED_BY_LABEL_FILTER, + "--filter", + "label=openshell.io/isolation-role=sandbox", + "--filter", + ]) .arg(sandbox_name_filter) .stdout(Stdio::piped()) .stderr(Stdio::piped()) diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index e30516bf09..e74888c84c 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -22,7 +22,7 @@ const BASE_IMAGE: &str = "ghcr.io/nvidia/openshell-community/sandboxes/base:late const READY_MARKER: &str = "podman-oci-identity-ready"; const OCI_UID: &str = "2345"; const OCI_GID: &str = "2346"; -const OCI_FALLBACK_POLICY: &str = r#"version: 1 +const OCI_FALLBACK_POLICY: &str = r"version: 1 filesystem_policy: include_workdir: true @@ -32,7 +32,7 @@ landlock: compatibility: best_effort network_policies: {} -"#; +"; struct ImageGuard { engine: ContainerEngine, @@ -128,7 +128,16 @@ fn run_engine(engine: &ContainerEngine, args: &[&str]) -> Result } fn sandbox_container_id(engine: &ContainerEngine, sandbox_name: &str) -> Result { + container_id_for_role(engine, sandbox_name, "sandbox") +} + +fn container_id_for_role( + engine: &ContainerEngine, + sandbox_name: &str, + role: &str, +) -> Result { let name_filter = format!("label=openshell.ai/sandbox-name={sandbox_name}"); + let role_filter = format!("label=openshell.io/isolation-role={role}"); let stdout = run_engine( engine, &[ @@ -138,6 +147,8 @@ fn sandbox_container_id(engine: &ContainerEngine, sandbox_name: &str) -> Result< "label=openshell.managed=true", "--filter", &name_filter, + "--filter", + &role_filter, ], )?; let ids = stdout @@ -233,5 +244,54 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { "Podman sandbox must launch the immutable image ID inspected before creation" ); + assert_isolated_pair(&image, &sandbox, &container_id).await; sandbox.cleanup().await; } + +async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, container_id: &str) { + let supervisor_id = container_id_for_role(&image.engine, &sandbox.name, "supervisor") + .expect("find separate supervisor companion"); + assert_ne!(supervisor_id, container_id); + for id in [container_id, &supervisor_id] { + let user = run_engine( + &image.engine, + &["inspect", "--format", "{{.Config.User}}", id], + ) + .unwrap(); + assert_eq!(user, format!("{OCI_UID}:{OCI_GID}")); + let caps = run_engine( + &image.engine, + &["inspect", "--format", "{{.EffectiveCaps}}", id], + ) + .unwrap(); + assert_eq!( + caps, "[]", + "neither container may have effective capabilities" + ); + } + let network = run_engine( + &image.engine, + &[ + "inspect", + "--format", + "{{.HostConfig.NetworkMode}}", + container_id, + ], + ) + .unwrap(); + assert_eq!(network, "none"); + let mounts = run_engine( + &image.engine, + &[ + "inspect", + "--format", + "{{range .Mounts}}{{println .Destination}}{{end}}", + container_id, + ], + ) + .unwrap(); + assert!(!mounts.contains("/etc/openshell/tls")); + assert!(!mounts.contains("/.openshell/supervisor")); + let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/runtime-descriptor.json"]).await.expect("workload cannot read either control credential set"); + assert!(posture.contains("0000000000000000")); +} diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index e7c47a251f..de34d6c10c 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -128,6 +128,8 @@ DRIVER_PID="" DRIVER_LOG="${OPENSHELL_PARITY_EXTERNAL_DRIVER_LOG_CAPTURE:-${WORKDIR}/podman-driver.log}" mkdir -p "$(dirname "${DRIVER_LOG}")" DRIVER_SOCKET="${WORKDIR}/compute-driver.sock" +DRIVER_DATA_HOME="${WORKDIR}/driver-data" +mkdir -p "${DRIVER_DATA_HOME}" E2E_NAMESPACE="" PODMAN_NETWORK_NAME="" PODMAN_NETWORK_MANAGED=0 @@ -182,9 +184,24 @@ cleanup() { for id in ${sandbox_ids}; do local sandbox_id sandbox_id="$(podman_cmd inspect --format '{{ index .Config.Labels "openshell.ai/sandbox-id" }}' "${id}" 2>/dev/null || true)" - podman_cmd rm -f "${id}" >/dev/null 2>&1 || true if [ -n "${sandbox_id}" ] && [ "${sandbox_id}" != "" ]; then + # Only the companion is attached to the test network. Remove it first + # (it depends on the workload user namespace), then locate the isolated + # network=none workload by this test sandbox's immutable label. + podman_cmd rm -f "openshell-supervisor-${sandbox_id}" >/dev/null 2>&1 || true + local workload_ids workload_id + workload_ids="$(podman_cmd ps -aq --filter "label=openshell.managed=true" \ + --filter "label=openshell.ai/sandbox-id=${sandbox_id}" \ + --filter "label=openshell.io/isolation-role=sandbox" 2>/dev/null || true)" + for workload_id in ${workload_ids}; do + podman_cmd rm -f "${workload_id}" >/dev/null 2>&1 || true + done + podman_cmd volume rm "openshell-channel-${sandbox_id}" >/dev/null 2>&1 || true podman_cmd volume rm -f "openshell-sandbox-${sandbox_id}-workspace" >/dev/null 2>&1 || true + local secret_prefix + for secret_prefix in openshell-token openshell-proxy-auth openshell-tls-ca openshell-tls-cert openshell-tls-key; do + podman_cmd secret rm "${secret_prefix}-${sandbox_id}" >/dev/null 2>&1 || true + done fi done fi @@ -339,6 +356,26 @@ resolve_podman_supervisor_image() { printf '%s\n' "openshell/supervisor:dev" } +resolve_podman_sandbox_runtime_image() { + if [ -n "${OPENSHELL_SANDBOX_RUNTIME_IMAGE:-}" ]; then + printf '%s\n' "${OPENSHELL_SANDBOX_RUNTIME_IMAGE}" + return 0 + fi + + if [ -n "${CI:-}" ]; then + if [ -z "${IMAGE_TAG:-}" ]; then + echo "ERROR: IMAGE_TAG must be set in CI when no Podman sandbox runtime image override is provided." >&2 + exit 2 + fi + + local registry="${OPENSHELL_REGISTRY:-ghcr.io/nvidia/openshell}" + printf '%s/sandbox:%s\n' "${registry%/}" "${IMAGE_TAG}" + return 0 + fi + + printf '%s\n' "openshell/sandbox:dev" +} + ensure_podman_supervisor_image() { local image=$1 @@ -433,6 +470,37 @@ ensure_podman_supervisor_image() { exit 2 } +ensure_podman_sandbox_runtime_image() { + local image=$1 + + if [ "${image}" = "openshell/sandbox:dev" ] \ + && [ -z "${OPENSHELL_SANDBOX_RUNTIME_IMAGE:-}" ] \ + && [ -z "${CI:-}" ]; then + echo "Building local Podman sandbox runtime image ${image}..." + with_podman_config env CONTAINER_ENGINE=podman IMAGE_TAG=dev \ + bash "${ROOT}/tasks/scripts/docker-build-image.sh" sandbox + if podman_cmd image exists "${image}" 2>/dev/null; then + return 0 + fi + + echo "ERROR: expected sandbox runtime image '${image}' after local build." >&2 + exit 2 + fi + + if podman_cmd image exists "${image}" 2>/dev/null; then + return 0 + fi + + echo "Pulling Podman sandbox runtime image ${image}..." + if podman_cmd pull "${image}"; then + return 0 + fi + + echo "ERROR: sandbox runtime image '${image}' is not available." >&2 + echo " Build it, push it, or set OPENSHELL_SANDBOX_RUNTIME_IMAGE to a pullable image." >&2 + exit 2 +} + if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ]; then case "${OPENSHELL_GATEWAY_ENDPOINT}" in http://*) ;; @@ -512,6 +580,10 @@ if ! [[ "${SUPERVISOR_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then exit 2 fi SUPERVISOR_BASE_IMAGE="$(awk '$1 == "FROM" { print $2; exit }' "${OPENSHELL_E2E_SUPERVISOR_DOCKERFILE:-${ROOT}/deploy/docker/Dockerfile.supervisor}")" +if ! podman_cmd image exists "${SUPERVISOR_BASE_IMAGE}" 2>/dev/null; then + echo "Pulling Podman supervisor base image ${SUPERVISOR_BASE_IMAGE}..." + podman_cmd pull "${SUPERVISOR_BASE_IMAGE}" +fi SUPERVISOR_BASE_IMAGE_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SUPERVISOR_BASE_IMAGE}")" SUPERVISOR_BASE_IMAGE_ID="${SUPERVISOR_BASE_IMAGE_ID#sha256:}" SUPERVISOR_BASE_IMAGE_DIGEST="$(podman_cmd image inspect --format '{{.Digest}}' "${SUPERVISOR_BASE_IMAGE}")" @@ -522,11 +594,16 @@ if ! [[ "${SUPERVISOR_BASE_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]] \ fi SUPERVISOR_PACKAGE_MANIFEST="${OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE:-${WORKDIR}/supervisor.packages.txt}" mkdir -p "$(dirname "${SUPERVISOR_PACKAGE_MANIFEST}")" -podman_cmd run --rm --network none --entrypoint /sbin/apk \ - "${SUPERVISOR_RUNTIME_IMAGE}" info -v | LC_ALL=C sort >"${SUPERVISOR_PACKAGE_MANIFEST}" +podman_cmd run --rm --network none --entrypoint /usr/bin/dpkg-query \ + "${SUPERVISOR_RUNTIME_IMAGE}" -W '-f=${binary:Package}=${Version}\n' \ + | LC_ALL=C sort >"${SUPERVISOR_PACKAGE_MANIFEST}" SUPERVISOR_PACKAGE_MANIFEST_SHA256="$(sha256sum "${SUPERVISOR_PACKAGE_MANIFEST}" | cut -d' ' -f1)" echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST}, base ${SUPERVISOR_BASE_IMAGE} ID ${SUPERVISOR_BASE_IMAGE_ID} digest ${SUPERVISOR_BASE_IMAGE_DIGEST}, packages ${SUPERVISOR_PACKAGE_MANIFEST_SHA256})" +SANDBOX_BOUNDARY_IMAGE="$(resolve_podman_sandbox_runtime_image)" +ensure_podman_sandbox_runtime_image "${SANDBOX_BOUNDARY_IMAGE}" +echo "Using Podman sandbox runtime image: ${SANDBOX_BOUNDARY_IMAGE}" + DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE_REQUEST="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" if [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = "1" ] \ @@ -651,16 +728,18 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then driver_tls_ca_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CA}" | cut -d' ' -f1)" driver_tls_cert_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CERT}" | cut -d' ' -f1)" driver_tls_key_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_KEY}" | cut -d' ' -f1)" - external_driver_environment="$(printf '{\"OPENSHELL_COMPUTE_DRIVER_SOCKET\":\"%s\",\"OPENSHELL_PODMAN_SOCKET\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE_PULL_POLICY\":\"%s\",\"OPENSHELL_HEALTH_CHECK_INTERVAL_SECS\":%s,\"OPENSHELL_GRPC_ENDPOINT\":\"%s\",\"OPENSHELL_GATEWAY_PORT\":%s,\"OPENSHELL_NETWORK_NAME\":\"%s\",\"OPENSHELL_STOP_TIMEOUT\":%s,\"OPENSHELL_SUPERVISOR_IMAGE\":\"%s\",\"OPENSHELL_PODMAN_TLS_CA\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_CERT\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_KEY\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_ENABLE_BIND_MOUNTS\":%s}' \ + external_driver_environment="$(printf '{\"XDG_DATA_HOME\":\"%s\",\"OPENSHELL_COMPUTE_DRIVER_SOCKET\":\"%s\",\"OPENSHELL_PODMAN_SOCKET\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE_PULL_POLICY\":\"%s\",\"OPENSHELL_HEALTH_CHECK_INTERVAL_SECS\":%s,\"OPENSHELL_GRPC_ENDPOINT\":\"%s\",\"OPENSHELL_GATEWAY_PORT\":%s,\"OPENSHELL_NETWORK_NAME\":\"%s\",\"OPENSHELL_STOP_TIMEOUT\":%s,\"OPENSHELL_SANDBOX_RUNTIME_IMAGE\":\"%s\",\"OPENSHELL_SUPERVISOR_IMAGE\":\"%s\",\"OPENSHELL_PODMAN_TLS_CA\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_CERT\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_KEY\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_ENABLE_BIND_MOUNTS\":%s}' \ + "${DRIVER_DATA_HOME}" \ "${DRIVER_SOCKET}" \ "${OPENSHELL_PODMAN_SOCKET:-}" \ - "${SANDBOX_RUNTIME_IMAGE}" \ + "${SANDBOX_IMAGE_REQUEST}" \ "${EXTERNAL_DRIVER_PULL_POLICY}" \ "${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ "${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ "${HOST_PORT}" \ "${PODMAN_NETWORK_NAME}" \ "${PODMAN_STOP_TIMEOUT_SECS}" \ + "${SANDBOX_BOUNDARY_IMAGE}" \ "${SUPERVISOR_RUNTIME_IMAGE}" \ "${EXTERNAL_DRIVER_TLS_CA}" \ "${driver_tls_ca_sha256}" \ @@ -670,7 +749,7 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then "${driver_tls_key_sha256}" \ "${EXTERNAL_DRIVER_ENABLE_BIND_MOUNTS}")" fi - printf '{"schema_version":%s,"gateway_port":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_base_runtime_image":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s","sandbox_client_image_alias":"%s","sandbox_client_image_alias_id":"%s","gateway_sha256_before_execution":"%s","cli_sha256_before_execution":"%s","conformance_sha256_before_execution":"%s","external_driver_sha256_before_execution":"%s","supervisor_sha256_before_execution":"%s","supervisor_dockerfile_sha256_before_execution":"%s","cli_trace_wrapper_sha256_before_execution":"%s","external_driver_grpc_endpoint":%s,"external_driver_host_gateway_ip":%s,"external_driver_userns":%s,"external_driver_spiffe":%s,"external_driver_proxy":%s,"external_driver_app_armor":%s,"external_driver_environment":%s}\n' \ + printf '{"schema_version":%s,"gateway_port":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_base_runtime_image":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s","sandbox_boundary_image":"%s","sandbox_client_image_alias":"%s","sandbox_client_image_alias_id":"%s","gateway_sha256_before_execution":"%s","cli_sha256_before_execution":"%s","conformance_sha256_before_execution":"%s","external_driver_sha256_before_execution":"%s","supervisor_sha256_before_execution":"%s","supervisor_dockerfile_sha256_before_execution":"%s","cli_trace_wrapper_sha256_before_execution":"%s","external_driver_grpc_endpoint":%s,"external_driver_host_gateway_ip":%s,"external_driver_userns":%s,"external_driver_spiffe":%s,"external_driver_proxy":%s,"external_driver_app_armor":%s,"external_driver_environment":%s}\n' \ "${CONFIG_SCHEMA_VERSION}" \ "${HOST_PORT}" \ "$([ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ] && printf true || printf false)" \ @@ -689,6 +768,7 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then "${SANDBOX_IMAGE_ID}" \ "${SANDBOX_IMAGE_DIGEST}" \ "${SANDBOX_RUNTIME_IMAGE}" \ + "${SANDBOX_BOUNDARY_IMAGE}" \ "${SANDBOX_CLIENT_IMAGE_ALIAS}" \ "${SANDBOX_CLIENT_IMAGE_ALIAS_ID}" \ "${OPENSHELL_E2E_EXPECTED_GATEWAY_SHA256:-}" \ @@ -712,15 +792,17 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then require_expected_sha256 "external compute driver" "${DRIVER_BIN}" \ "${OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256:-}" env -i \ + XDG_DATA_HOME="${DRIVER_DATA_HOME}" \ OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ - OPENSHELL_SANDBOX_IMAGE="${SANDBOX_RUNTIME_IMAGE}" \ + OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE_REQUEST}" \ OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="${EXTERNAL_DRIVER_PULL_POLICY}" \ OPENSHELL_HEALTH_CHECK_INTERVAL_SECS="${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ OPENSHELL_GRPC_ENDPOINT="${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ + OPENSHELL_SANDBOX_RUNTIME_IMAGE="${SANDBOX_BOUNDARY_IMAGE}" \ OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_RUNTIME_IMAGE}" \ OPENSHELL_PODMAN_TLS_CA="${EXTERNAL_DRIVER_TLS_CA}" \ OPENSHELL_PODMAN_TLS_CERT="${EXTERNAL_DRIVER_TLS_CERT}" \ diff --git a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py index 6e8009b759..43b3b4aa2c 100644 --- a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py +++ b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py @@ -22,6 +22,7 @@ IMAGE_ID = "3" * 64 IMAGE_DIGEST = f"sha256:{'4' * 64}" RUNTIME_IMAGE = f"localhost/openshell/supervisor@{IMAGE_DIGEST}" +BOUNDARY_IMAGE = f"localhost/openshell/sandbox@{IMAGE_DIGEST}" BASE_RUNTIME_IMAGE = f"docker.io/library/alpine@{IMAGE_DIGEST}" @@ -126,6 +127,7 @@ def create_variant( "sandbox_image_id": IMAGE_ID, "sandbox_image_digest": IMAGE_DIGEST, "sandbox_runtime_image": "example.invalid/sandbox@" + IMAGE_DIGEST, + "sandbox_boundary_image": BOUNDARY_IMAGE, "sandbox_client_image_alias": "example.invalid/sandbox:latest", "sandbox_client_image_alias_id": IMAGE_ID, "gateway_sha256_before_execution": result["gateway_sha256"], @@ -146,6 +148,7 @@ def create_variant( "external_driver_proxy": False, "external_driver_app_armor": False, "external_driver_environment": { + "XDG_DATA_HOME": f"/tmp/{variant}-driver-data", "OPENSHELL_COMPUTE_DRIVER_SOCKET": f"/tmp/{variant}.sock", "OPENSHELL_PODMAN_SOCKET": f"/tmp/{variant}-podman.sock", "OPENSHELL_SANDBOX_IMAGE": "example.invalid/sandbox@" + IMAGE_DIGEST, @@ -155,6 +158,7 @@ def create_variant( "OPENSHELL_GATEWAY_PORT": 18181, "OPENSHELL_NETWORK_NAME": f"{variant}-network", "OPENSHELL_STOP_TIMEOUT": 15, + "OPENSHELL_SANDBOX_RUNTIME_IMAGE": BOUNDARY_IMAGE, "OPENSHELL_SUPERVISOR_IMAGE": RUNTIME_IMAGE, "OPENSHELL_PODMAN_TLS_CA": { "path": f"/tmp/{variant}-pki/ca.crt", @@ -178,7 +182,7 @@ def create_variant( (results_dir / f"{variant}.log").write_text( f"CLI conformance run ID: fixture\n" f"gateway preflight connected: gateway=fixture, authentication=authenticated\n" - f"{lifecycle}\n{RUNTIME_IMAGE} {BASE_RUNTIME_IMAGE} example.invalid/sandbox@{IMAGE_DIGEST} example.invalid/sandbox:latest " + f"{lifecycle}\n{RUNTIME_IMAGE} {BOUNDARY_IMAGE} {BASE_RUNTIME_IMAGE} example.invalid/sandbox@{IMAGE_DIGEST} example.invalid/sandbox:latest " f'{IMAGE_ID} {IMAGE_DIGEST} {package_hash}\n"passed": true\n', encoding="utf-8", ) diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index a862db2f2f..5717b0e884 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -253,10 +253,14 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Supervisor cannot call back: check callback endpoint and gateway logs. -- A sandbox with explicit `protocol: tcp` endpoints fails before readiness: - inspect supervisor logs for policy DNS port-53 binding, synthetic-route, or - nftables redirect failures. Rootless Podman must provide these primitives - inside the supervisor-owned nested network namespace; setup fails closed. +- Inspect both Podman containers for the sandbox: the `sandbox` isolation role + must have network mode `none`; the `supervisor` role owns gateway callbacks + and egress. Both run non-root with all capabilities dropped. Check the private + channel volume and shared user-namespace mapping if authentication fails. +- If a sandbox fails before readiness, inspect its unprivileged enforcement + probe and the companion supervisor's private health check. Do not add + capabilities, attach a workload network, or disable the runtime seccomp + profile. There is no sandbox nftables or nested-network setup to repair. - Gateway exits before becoming healthy with a callback-listener discovery error: inspect `podman info --debug`, the configured Podman network, and the host's IPv4 default route. Rootless pasta uses the private source address