From 997b5d23d64561fc6108ae68eb516909c1f8260a Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 16:22:21 -0700 Subject: [PATCH 1/2] fix(kubernetes): scope resource admission RBAC Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 7 +- architecture/compute-runtimes.md | 6 +- crates/openshell-driver-kubernetes/README.md | 4 +- .../openshell-driver-kubernetes/src/driver.rs | 74 +------------------ .../src/resource_admission.rs | 72 +++++++----------- .../openshell-workspace/templates/role.yaml | 4 +- .../tests/workspace_test.yaml | 18 +++++ .../helm/openshell/templates/clusterrole.yaml | 2 +- deploy/helm/openshell/templates/role.yaml | 4 +- .../openshell/tests/clusterrole_test.yaml | 23 ++++++ .../tests/sandbox_namespace_test.yaml | 18 +++++ docs/reference/gateway-config.mdx | 41 +++++----- skills/debug-openshell-cluster/SKILL.md | 15 ++-- 13 files changed, 129 insertions(+), 159 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 2df971436f..cb3023f476 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -72,9 +72,10 @@ mise run helm:skaffold:run Resource admission defaults to enabled and caller driver config to disabled. Driver-config scenarios need an explicit `allowDriverConfig` opt-in; external attachments also need administrator-controlled approval labels in the target -namespace. GPU attachments are exempt from labels. Managed workspace image-pull -Secrets are copied from an approved source in the gateway namespace; do not grant -approval to the gateway database PVC or disable admission to make tests pass. +namespace. GPU attachments and operator-selected image-pull Secrets are exempt +from labels. Managed workspace image-pull Secrets are copied from the configured +source in the gateway namespace; do not grant approval to the gateway database +PVC or disable admission to make tests pass. The Skaffold flow builds distinct `gateway`, `sandbox`, and `supervisor` images and deploys the OpenShell Helm chart. The Kubernetes driver creates a diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 44415009c8..3f28c3f0a4 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -465,9 +465,9 @@ management. RBAC uses a namespace-scoped Role. Each new namespace receives a ServiceAccount and the configured gateway-only SSH ingress NetworkPolicy. Configured image-pull Secrets are copied from the driver's source namespace on every sandbox create so registry credential -rotations propagate. Resource admission first validates the source as shared -operator infrastructure, and copies carry gateway and workspace ownership -labels; an unrelated existing target is never adopted. The namespace also copies +rotations propagate. Their names are operator-selected gateway configuration, +not caller attachments. Copies carry gateway and workspace ownership labels; an +unrelated existing target is never adopted. The namespace also copies OpenShift SCC UID-range and supplemental-group annotations from the gateway namespace when present. The driver deletes the namespace during workspace deletion. The workspace remains durably `Terminating` until the Kubernetes API diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 6e1578cee4..d21a6c1ff1 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -5,8 +5,8 @@ Kubernetes-backed compute driver for OpenShell cluster deployments. Caller driver config is disabled by default. External resource references need administrator-controlled approval labels in every workspace mode, including before restart and scheduling-gate release. GPU devices are temporarily exempt. -Managed workspace image-pull Secrets are copied only after the configured source -Secret passes shared-resource admission; copies carry gateway ownership metadata. +Image-pull Secrets are operator-selected gateway configuration rather than caller +attachments. Managed workspace copies carry gateway ownership metadata. See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). The driver uses the Kubernetes API to create, delete, fetch, and watch sandbox diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 356fdc249d..5d072a0f21 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1306,28 +1306,6 @@ impl KubernetesComputeDriver { } }; - self.config - .resource_admission - .admit_shared( - source - .metadata - .labels - .as_ref() - .into_iter() - .flat_map(|labels| labels.iter()), - ) - .map_err(|error| { - admission_error(tonic::Status::new( - error.code(), - format!( - "Secret '{}/{}': {}", - self.config.namespace, - secret_name, - error.message() - ), - )) - })?; - let existing = tokio::time::timeout(KUBE_API_TIMEOUT, target_api.get_opt(secret_name)) .await .map_err(|_| { @@ -11070,7 +11048,7 @@ mod tests { } #[tokio::test] - async fn managed_image_pull_secret_admits_source_before_copying() { + async fn managed_image_pull_secret_copies_operator_selected_source() { let source_path = "/api/v1/namespaces/openshell/secrets/regcred"; let target_path = "/api/v1/namespaces/managed-team-a/secrets/regcred"; let copied = serde_json::json!({ @@ -11101,7 +11079,7 @@ mod tests { "metadata": { "name": "regcred", "namespace": "openshell", - "labels": {"openshell.ai/sandbox-attachable": "true"} + "labels": {} }, "type": "kubernetes.io/dockerconfigjson", "data": { ".dockerconfigjson": "e30=" } @@ -11150,56 +11128,10 @@ mod tests { driver .ensure_image_pull_secrets("managed-team-a", "team-a") .await - .expect("approved source should be copied"); + .expect("operator-selected source should be copied"); assert!(steps.lock().unwrap().is_empty()); } - #[tokio::test] - async fn managed_image_pull_secret_rejects_unapproved_source_without_copying() { - let service = tower::service_fn( - move |request: http::Request| async move { - assert_eq!(request.method(), http::Method::GET); - assert_eq!( - request.uri().path(), - "/api/v1/namespaces/openshell/secrets/regcred" - ); - Ok::<_, std::convert::Infallible>(kube_test_response( - http::StatusCode::OK, - serde_json::json!({ - "apiVersion": "v1", - "kind": "Secret", - "metadata": {"name": "regcred", "namespace": "openshell"}, - "type": "kubernetes.io/dockerconfigjson", - "data": { ".dockerconfigjson": "e30=" } - }), - )) - }, - ); - let client = Client::new(service, "openshell"); - let driver = KubernetesComputeDriver { - client: client.clone(), - watch_client: client, - sandbox_api_version: Arc::new(OnceCell::new()), - config: KubernetesComputeConfig { - namespace: "openshell".into(), - gateway_id: "gateway-a".into(), - image_pull_secrets: vec!["regcred".into()], - ..Default::default() - }, - operator_allowlist: None, - }; - - let error = driver - .ensure_image_pull_secrets("managed-team-a", "team-a") - .await - .expect_err("unapproved source must not be copied"); - assert!(matches!(error, KubernetesDriverError::Precondition(_))); - assert!( - error.to_string().contains("Secret 'openshell/regcred'"), - "unexpected error: {error}" - ); - } - #[test] fn namespace_owned_with_correct_labels() { let labels = BTreeMap::from([ diff --git a/crates/openshell-driver-kubernetes/src/resource_admission.rs b/crates/openshell-driver-kubernetes/src/resource_admission.rs index fa288c4f80..339572d1f2 100644 --- a/crates/openshell-driver-kubernetes/src/resource_admission.rs +++ b/crates/openshell-driver-kubernetes/src/resource_admission.rs @@ -110,9 +110,9 @@ fn inventory(spec: &Value, private_secret: &str) -> Result, { return Err(deny()); } - for secret in spec["imagePullSecrets"].as_array().into_iter().flatten() { - reference(&mut refs, "Secret", secret["name"].as_str(), Scope::Shared); - } + // Image-pull Secrets are selected by gateway configuration rather than by + // the sandbox caller. The kubelet resolves them; they are not workload data + // attachments and do not participate in caller resource admission. for volume in spec["volumes"].as_array().into_iter().flatten() { let object = volume.as_object().ok_or_else(deny)?; let sources: Vec<_> = object.keys().filter(|key| key.as_str() != "name").collect(); @@ -127,50 +127,27 @@ fn inventory(spec: &Value, private_secret: &str) -> Result, volume["persistentVolumeClaim"]["claimName"].as_str(), Scope::Workspace, ), - "secret" => { - let name = volume["secret"]["secretName"].as_str(); - if name != Some(private_secret) { - reference(&mut refs, "Secret", name, Scope::Workspace); - } - } - "configMap" => reference( - &mut refs, - "ConfigMap", - volume["configMap"]["name"].as_str(), - Scope::Workspace, - ), + "secret" if volume["secret"]["secretName"].as_str() == Some(private_secret) => {} + // The typed Kubernetes driver config does not expose Secret or + // ConfigMap volumes. The closed-set fallback rejects them instead + // of expanding gateway RBAC for attachments callers cannot request. _ => return Err(deny()), } } for field in ["containers", "initContainers", "ephemeralContainers"] { for container in spec[field].as_array().into_iter().flatten() { - for env in container["envFrom"].as_array().into_iter().flatten() { - reference( - &mut refs, - "Secret", - env["secretRef"]["name"].as_str(), - Scope::Workspace, - ); - reference( - &mut refs, - "ConfigMap", - env["configMapRef"]["name"].as_str(), - Scope::Workspace, - ); + if container["envFrom"] + .as_array() + .is_some_and(|env| !env.is_empty()) + { + return Err(deny()); } for env in container["env"].as_array().into_iter().flatten() { - reference( - &mut refs, - "Secret", - env["valueFrom"]["secretKeyRef"]["name"].as_str(), - Scope::Workspace, - ); - reference( - &mut refs, - "ConfigMap", - env["valueFrom"]["configMapKeyRef"]["name"].as_str(), - Scope::Workspace, - ); + if !env["valueFrom"]["secretKeyRef"].is_null() + || !env["valueFrom"]["configMapKeyRef"].is_null() + { + return Err(deny()); + } } for field in ["requests", "limits"] { for (resource, _) in container["resources"][field] @@ -207,8 +184,6 @@ pub async fn admit( for reference in inventory(spec, private_secret)? { let (group, version, plural, cluster) = match reference.kind { "PersistentVolumeClaim" => ("", "v1", "persistentvolumeclaims", false), - "Secret" => ("", "v1", "secrets", false), - "ConfigMap" => ("", "v1", "configmaps", false), "RuntimeClass" => ("node.k8s.io", "v1", "runtimeclasses", true), "PriorityClass" => ("scheduling.k8s.io", "v1", "priorityclasses", true), _ => unreachable!("closed resource inventory"), @@ -479,13 +454,12 @@ mod tests { } #[test] - fn inventories_all_containers_and_reference_aliases() { + fn inventories_supported_external_resources() { let pod = serde_json::json!({"automountServiceAccountToken":false,"runtimeClassName":"r","priorityClassName":"p", "volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"gateway-db","readOnly":true}}], - "initContainers":[{"envFrom":[{"secretRef":{"name":"secret"}}]}], - "containers":[{"env":[{"valueFrom":{"configMapKeyRef":{"name":"config"}}}]}]}); + "imagePullSecrets":[{"name":"regcred"}]}); let refs = inventory(&pod, "private").unwrap(); - assert_eq!(refs.len(), 5); + assert_eq!(refs.len(), 3); assert!( refs.iter() .any(|r| r.name == "gateway-db" && r.scope == Scope::Workspace) @@ -497,9 +471,13 @@ mod tests { } #[test] fn rejects_unsupported_volume_sources_but_allows_gpu() { - for kind in ["hostPath", "csi", "projected", "image"] { + for kind in ["hostPath", "csi", "projected", "image", "configMap"] { assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"volumes":[{"name":"x",kind:{}}]}), "private").is_err()); } + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"volumes":[{"name":"x","secret":{"secretName":"external"}}]}), "private").is_err()); + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"volumes":[{"name":"x","secret":{"secretName":"private"}}]}), "private").is_ok()); + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"containers":[{"envFrom":[{"secretRef":{"name":"external"}}]}]}), "private").is_err()); + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"containers":[{"env":[{"valueFrom":{"configMapKeyRef":{"name":"external"}}}]}]}), "private").is_err()); assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"containers":[{"resources":{"limits":{"nvidia.com/gpu":"1"}}}]}), "private").is_ok()); } #[test] diff --git a/deploy/helm/openshell-workspace/templates/role.yaml b/deploy/helm/openshell-workspace/templates/role.yaml index 3df8928496..07e3b422e3 100644 --- a/deploy/helm/openshell-workspace/templates/role.yaml +++ b/deploy/helm/openshell-workspace/templates/role.yaml @@ -10,9 +10,9 @@ metadata: labels: {{- include "openshell-workspace.labels" . | nindent 4 }} rules: - # Metadata-only application admission still requires Kubernetes get permission. + # Metadata-only admission of caller-selected PVCs requires get permission. - apiGroups: [""] - resources: ["persistentvolumeclaims", "configmaps", "secrets"] + resources: ["persistentvolumeclaims"] verbs: ["get"] - apiGroups: - agents.x-k8s.io diff --git a/deploy/helm/openshell-workspace/tests/workspace_test.yaml b/deploy/helm/openshell-workspace/tests/workspace_test.yaml index 2f71920eaf..b92d80a300 100644 --- a/deploy/helm/openshell-workspace/tests/workspace_test.yaml +++ b/deploy/helm/openshell-workspace/tests/workspace_test.yaml @@ -20,6 +20,24 @@ tests: - equal: path: metadata.namespace value: app-a + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] - it: binds the shared gateway service account template: templates/rolebinding.yaml diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 1df5c52467..aeba20ae9b 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -17,7 +17,7 @@ rules: verbs: ["get"] {{- if ne $workspaceMode "shared" }} - apiGroups: [""] - resources: ["persistentvolumeclaims", "configmaps", "secrets"] + resources: ["persistentvolumeclaims"] verbs: ["get"] {{- end }} # Validate projected ServiceAccount tokens during sandbox bootstrap and diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 365ef3e9c7..f11dde43be 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -10,9 +10,9 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: - # Metadata-only application admission still requires Kubernetes get permission. + # Metadata-only admission of caller-selected PVCs requires get permission. - apiGroups: [""] - resources: ["persistentvolumeclaims", "configmaps", "secrets"] + resources: ["persistentvolumeclaims"] verbs: ["get"] - apiGroups: - agents.x-k8s.io diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index 9639232c32..1f00c206eb 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -26,6 +26,29 @@ tests: resources: ["networkpolicies"] verbs: ["get", "create", "patch", "update"] + - it: limits namespaced admission reads to PVCs + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - it: preserves sandbox-runtime fence permissions when gateway isolation is disabled set: server.drivers.kubernetes.workspaceMode: managed diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 6ec7ae4d65..9454ceaba9 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -74,6 +74,24 @@ tests: apiGroups: [""] resources: ["secrets"] verbs: ["create", "list", "delete"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] - it: uses explicit sandboxNamespace for sandbox RoleBinding template: templates/rolebinding.yaml diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index a45da33fea..2e4f086dd6 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -574,26 +574,27 @@ metadata: openshell.ai/sandbox-attachable-workspace: "team-a" ``` -Kubernetes checks PVCs, RuntimeClasses, PriorityClasses, and credential/config -references selected through the OpenShell-owned Pod template. Docker and Podman -check engine volume labels. Read-only mounts still require approval. Namespaces -and operator-selected defaults do not waive checks. Newly provisioned private -storage and bootstrap material use driver ownership checks instead. - -RuntimeClasses, PriorityClasses, and image-pull Secrets are shared operator -infrastructure. They must match every fixed required label, but `${workspace}` -entries do not apply to them. PVCs and Secret or ConfigMap data exposed to a -sandbox must also match the resolved workspace label. Kubernetes API-server and -admission-webhook mutations of the live Pod are trusted cluster-operator behavior -and are not used as Workspace User authorization inputs. - -With Kubernetes admission enabled, configured image-pull Secrets in the gateway -namespace must carry the fixed approval labels. In managed mode OpenShell verifies -that source, creates the workspace namespace, and copies the Secret with gateway -ownership metadata. An existing target Secret is updated only when its ownership -metadata matches the gateway and workspace. Operator mode still requires the -approved Secret in each operator-managed namespace. Automatic service-account -tokens and projected token volumes are prohibited. +Kubernetes checks caller-selected PVCs, RuntimeClasses, and PriorityClasses +through the OpenShell-owned Pod template. Docker and Podman check engine volume +labels. Read-only mounts still require approval. Namespaces do not grant +approval. Newly provisioned private storage and bootstrap material use driver +ownership checks instead. + +RuntimeClasses and PriorityClasses are shared operator infrastructure. They must +match every fixed required label, but `${workspace}` entries do not apply to +them. Kubernetes API-server and admission-webhook mutations of the live Pod are +trusted cluster-operator behavior and are not used as Workspace User +authorization inputs. + +Image-pull Secrets are selected by gateway configuration, not by sandbox +callers, and do not participate in resource admission. In managed mode OpenShell +copies each explicitly configured Secret into the workspace namespace with +gateway ownership metadata. An existing target Secret is updated only when its +ownership metadata matches the gateway and workspace. In shared and operator +modes, the configured Secret must already exist in the sandbox namespace. The +typed Kubernetes driver config does not support caller-selected Secret or +ConfigMap volumes or environment references. Automatic service-account tokens +and projected token volumes are prohibited. GPU device attachments are temporarily exempt from label admission. Existing GPU validation still applies. Explicit GPU device settings inside driver JSON diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index e331c605e9..aea5c02607 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -94,14 +94,13 @@ Before debugging the compute platform, inspect gateway logs for failures in depe For resource-admission failures, distinguish disabled caller driver config from missing resource approval. Helm defaults `server.drivers.kubernetes.allowDriverConfig` -to false and `resourceAdmission.enabled` to true. Existing PVCs, Secrets, -ConfigMaps, RuntimeClasses, and PriorityClasses need matching administrator-owned -labels; namespace membership and read-only access do not grant approval. Inspect -metadata only when diagnosing credentials. GPU devices do not need labels. -RuntimeClasses, PriorityClasses, and image-pull Secrets match fixed approval -labels but do not use the workspace placeholder. In managed mode, inspect the -approved source image-pull Secret in the gateway namespace and the gateway-owned -copy in the workspace namespace. Legacy workloads without admission provenance need recreation. Do not +to false and `resourceAdmission.enabled` to true. Existing PVCs, RuntimeClasses, +and PriorityClasses need matching administrator-owned labels; namespace +membership and read-only access do not grant approval. GPU devices and +operator-selected image-pull Secrets do not need admission labels. In managed +mode, inspect the configured source image-pull Secret in the gateway namespace +and the gateway-owned copy in the workspace namespace. Legacy workloads without +admission provenance need recreation. Do not automatically label control-plane resources or disable enforcement as a repair. For out-of-tree compute drivers, also check that their versioned admission-policy From 36371f00dab58a711607c33496a4a473e9bbdbdd Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 16:33:35 -0700 Subject: [PATCH 2/2] fix(helm): gate PVC admission reads Signed-off-by: Drew Newberry --- deploy/helm/openshell-workspace/README.md | 6 +++++- deploy/helm/openshell-workspace/README.md.gotmpl | 5 ++++- .../helm/openshell-workspace/templates/role.yaml | 2 ++ .../openshell-workspace/tests/workspace_test.yaml | 14 +++++++++++++- deploy/helm/openshell-workspace/values.yaml | 3 +++ deploy/helm/openshell/templates/clusterrole.yaml | 2 +- deploy/helm/openshell/templates/role.yaml | 2 ++ deploy/helm/openshell/tests/clusterrole_test.yaml | 12 ++++++++++++ .../openshell/tests/sandbox_namespace_test.yaml | 14 +++++++++++++- docs/kubernetes/setup.mdx | 5 +++++ docs/reference/gateway-config.mdx | 7 ++++--- 11 files changed, 64 insertions(+), 8 deletions(-) diff --git a/deploy/helm/openshell-workspace/README.md b/deploy/helm/openshell-workspace/README.md index 96ae4ca6e4..e3a21b3b34 100644 --- a/deploy/helm/openshell-workspace/README.md +++ b/deploy/helm/openshell-workspace/README.md @@ -27,13 +27,17 @@ helm install openshell-workspace ./deploy/helm/openshell-workspace \ Keep `sandboxServiceAccount.name` aligned with the gateway chart's `sandboxServiceAccount.name`. The defaults for both charts are -`openshell-sandbox`. +`openshell-sandbox`. If the gateway enables +`server.drivers.kubernetes.allowDriverConfig`, also set +`gateway.allowDriverConfig=true` in every workspace release so caller-selected +PVCs can be admitted. ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| | fullnameOverride | string | `""` | Override the full generated resource name. | +| gateway.allowDriverConfig | bool | `false` | Grant PVC metadata reads required when the gateway enables caller driver config. Keep this aligned with server.drivers.kubernetes.allowDriverConfig in the gateway chart. | | gateway.networkPolicy.podSelector | object | `{"app.kubernetes.io/instance":"openshell","app.kubernetes.io/name":"openshell"}` | Labels selecting gateway pods allowed to reach sandbox SSH. | | gateway.serviceAccount.name | string | `"openshell"` | Name of the shared gateway ServiceAccount. | | gateway.serviceAccount.namespace | string | `"openshell"` | Namespace containing the shared gateway ServiceAccount. | diff --git a/deploy/helm/openshell-workspace/README.md.gotmpl b/deploy/helm/openshell-workspace/README.md.gotmpl index a233eef165..a691aba4a2 100644 --- a/deploy/helm/openshell-workspace/README.md.gotmpl +++ b/deploy/helm/openshell-workspace/README.md.gotmpl @@ -27,7 +27,10 @@ helm install openshell-workspace ./deploy/helm/openshell-workspace \ Keep `sandboxServiceAccount.name` aligned with the gateway chart's `sandboxServiceAccount.name`. The defaults for both charts are -`openshell-sandbox`. +`openshell-sandbox`. If the gateway enables +`server.drivers.kubernetes.allowDriverConfig`, also set +`gateway.allowDriverConfig=true` in every workspace release so caller-selected +PVCs can be admitted. {{ template "chart.valuesSection" . }} {{ template "helm-docs.versionFooter" . }} diff --git a/deploy/helm/openshell-workspace/templates/role.yaml b/deploy/helm/openshell-workspace/templates/role.yaml index 07e3b422e3..bd0e33bad6 100644 --- a/deploy/helm/openshell-workspace/templates/role.yaml +++ b/deploy/helm/openshell-workspace/templates/role.yaml @@ -10,10 +10,12 @@ metadata: labels: {{- include "openshell-workspace.labels" . | nindent 4 }} rules: + {{- if .Values.gateway.allowDriverConfig }} # Metadata-only admission of caller-selected PVCs requires get permission. - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get"] + {{- end }} - apiGroups: - agents.x-k8s.io resources: diff --git a/deploy/helm/openshell-workspace/tests/workspace_test.yaml b/deploy/helm/openshell-workspace/tests/workspace_test.yaml index b92d80a300..7358e7de38 100644 --- a/deploy/helm/openshell-workspace/tests/workspace_test.yaml +++ b/deploy/helm/openshell-workspace/tests/workspace_test.yaml @@ -20,7 +20,7 @@ tests: - equal: path: metadata.namespace value: app-a - - contains: + - notContains: path: rules content: apiGroups: [""] @@ -39,6 +39,18 @@ tests: resources: ["configmaps"] verbs: ["get"] + - it: grants PVC admission reads only when caller driver config is enabled + template: templates/role.yaml + set: + gateway.allowDriverConfig: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - it: binds the shared gateway service account template: templates/rolebinding.yaml set: diff --git a/deploy/helm/openshell-workspace/values.yaml b/deploy/helm/openshell-workspace/values.yaml index 2c52b0460d..069e573e2f 100644 --- a/deploy/helm/openshell-workspace/values.yaml +++ b/deploy/helm/openshell-workspace/values.yaml @@ -7,6 +7,9 @@ nameOverride: "" fullnameOverride: "" gateway: + # -- Grant PVC metadata reads required when the gateway enables caller driver config. + # Keep this aligned with server.drivers.kubernetes.allowDriverConfig in the gateway chart. + allowDriverConfig: false serviceAccount: # -- Name of the shared gateway ServiceAccount. name: openshell diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index aeba20ae9b..8bd68eeb5c 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -15,7 +15,7 @@ rules: - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["get"] - {{- if ne $workspaceMode "shared" }} + {{- if and (ne $workspaceMode "shared") .Values.server.drivers.kubernetes.allowDriverConfig }} - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get"] diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index f11dde43be..d43f5de0ab 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -10,10 +10,12 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: + {{- if .Values.server.drivers.kubernetes.allowDriverConfig }} # Metadata-only admission of caller-selected PVCs requires get permission. - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get"] + {{- end }} - apiGroups: - agents.x-k8s.io resources: diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index 1f00c206eb..528efd7cba 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -29,6 +29,7 @@ tests: - it: limits namespaced admission reads to PVCs set: server.drivers.kubernetes.workspaceMode: managed + server.drivers.kubernetes.allowDriverConfig: true asserts: - contains: path: rules @@ -49,6 +50,17 @@ tests: resources: ["configmaps"] verbs: ["get"] + - it: omits PVC admission reads when caller driver config is disabled + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - it: preserves sandbox-runtime fence permissions when gateway isolation is disabled set: server.drivers.kubernetes.workspaceMode: managed diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 9454ceaba9..36c2276e91 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -74,7 +74,7 @@ tests: apiGroups: [""] resources: ["secrets"] verbs: ["create", "list", "delete"] - - contains: + - notContains: path: rules content: apiGroups: [""] @@ -93,6 +93,18 @@ tests: resources: ["configmaps"] verbs: ["get"] + - it: grants PVC admission reads only when caller driver config is enabled + template: templates/role.yaml + set: + server.drivers.kubernetes.allowDriverConfig: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - it: uses explicit sandboxNamespace for sandbox RoleBinding template: templates/rolebinding.yaml set: diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 43e221b824..494e14aa22 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -132,6 +132,11 @@ provisioned namespaces, use `workspaceMode=operator`, configure exactly one of `operatorNamespaceLabel` or `operatorNamespaceFile`, and install the workspace chart in every allowlisted namespace. +If you enable `server.drivers.kubernetes.allowDriverConfig` on the gateway, +also set `gateway.allowDriverConfig=true` on every workspace release. This +grants the namespace-scoped PVC metadata read needed to admit caller-selected +PVC mounts. Leave both values disabled when callers do not need driver config. + ## Wait for the gateway to be ready ```shell diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 2e4f086dd6..53bb8ceef0 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -651,9 +651,10 @@ calls until gateway and driver configuration agree again. Helm exposes `server.drivers.kubernetes.allowDriverConfig`, `resourceAdmission.enabled`, and `resourceAdmission.requiredLabels`. The label map defaults to `null`, which uses code defaults without merging those keys -into a custom map. Admission reads need Kubernetes `get` permissions even -though only metadata is requested. Never mark gateway database storage as -attachable. +into a custom map. The Helm chart grants PVC metadata reads only when +`allowDriverConfig` is enabled. Split workspace releases must set the matching +`gateway.allowDriverConfig=true` workspace-chart value. Never mark gateway +database storage as attachable. Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields.