From 60660a0451d007c3d1af60c73be82a69df6c956d Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Tue, 23 Jun 2026 14:48:36 +0200 Subject: [PATCH] NO-JIRA: add kms plugin to preflight pod Signed-off-by: Thomas Jungblut --- .../preflight/assets/kms-preflight-pod.yaml | 6 - .../encryption/kms/preflight/checker.go | 2 + .../encryption/kms/preflight/deployer.go | 43 ++++- .../encryption/kms/preflight/deployer_test.go | 175 +++++++++++++++++- .../kms/preflight/pod_template_test.go | 6 - 5 files changed, 208 insertions(+), 24 deletions(-) diff --git a/pkg/operator/encryption/kms/preflight/assets/kms-preflight-pod.yaml b/pkg/operator/encryption/kms/preflight/assets/kms-preflight-pod.yaml index d72dda71fb..eb30587d91 100644 --- a/pkg/operator/encryption/kms/preflight/assets/kms-preflight-pod.yaml +++ b/pkg/operator/encryption/kms/preflight/assets/kms-preflight-pod.yaml @@ -41,9 +41,3 @@ spec: requests: memory: 50Mi cpu: 5m - volumeMounts: - - name: kms-plugin-socket - mountPath: /var/run/kmsplugin - volumes: - - name: kms-plugin-socket - emptyDir: {} diff --git a/pkg/operator/encryption/kms/preflight/checker.go b/pkg/operator/encryption/kms/preflight/checker.go index 4570218fb8..89849e3706 100644 --- a/pkg/operator/encryption/kms/preflight/checker.go +++ b/pkg/operator/encryption/kms/preflight/checker.go @@ -22,6 +22,8 @@ const healthzOK = "ok" // Status, Encrypt, and Decrypt on the kmsservice.Service interface. // this is the same interface the apiserver uses. type checker struct { + // TODO(thomas): configure sockets like the health monitor cmd + // TODO(thomas): get the env variables/args for the pod/namespace service kmsservice.Service randReader io.Reader statusTimeout time.Duration diff --git a/pkg/operator/encryption/kms/preflight/deployer.go b/pkg/operator/encryption/kms/preflight/deployer.go index 05e912faee..d6c96756a6 100644 --- a/pkg/operator/encryption/kms/preflight/deployer.go +++ b/pkg/operator/encryption/kms/preflight/deployer.go @@ -5,19 +5,24 @@ import ( "fmt" "time" + "github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + + "github.com/openshift/library-go/pkg/operator/encryption/encryptiondata" ) const ( - preflightPodName = "kms-preflight" + preflightPodName = "kms-preflight" + preflightCheckContainerName = "kms-preflight-check" + preflightEncryptionConfigSecretName = "kms-preflight-encryption-config" ) type KMSPreflightDeployer interface { - // Deploy creates the preflight checker with the given config hash - Deploy(ctx context.Context, configHash string) error + // Deploy creates the preflight checker with the given config hash and the encryption config. + Deploy(ctx context.Context, configHash string, encryptionConfig *encryptiondata.Config) error Status(ctx context.Context) (corev1.PodStatus, error) // Cleanup cleans up anything previously created. Cleanup(ctx context.Context) error @@ -36,7 +41,14 @@ type PodPreflightDeployer struct { kmsCallTimeout time.Duration } -func (d *PodPreflightDeployer) Deploy(ctx context.Context, configHash string) error { +func (d *PodPreflightDeployer) Deploy(ctx context.Context, configHash string, encryptionConfig *encryptiondata.Config) error { + if configHash == "" { + return fmt.Errorf("configHash is empty") + } + if encryptionConfig == nil { + return fmt.Errorf("encryptionConfig is nil") + } + pod, err := generatePodTemplate( preflightPodName, d.namespace, @@ -49,10 +61,24 @@ func (d *PodPreflightDeployer) Deploy(ctx context.Context, configHash string) er return fmt.Errorf("failed to generate preflight pod template: %w", err) } - // TODO(thomas): inject KMS plugin sidecar container into pod.Spec + secret, err := encryptiondata.ToSecret(d.namespace, preflightEncryptionConfigSecretName, encryptionConfig) + if err != nil { + return fmt.Errorf("failed to generate proposed encryption config secret: %w", err) + } + + err = pluginlifecycle.NewKMSPluginBuilder(). + FromEncryptionConfig(secret.Name, encryptionConfig). + // TODO(thomas): add a "WithPreflight" option to fill the socket names and other arguments (like health cmd) + Apply(&pod.Spec, preflightCheckContainerName) + if err != nil { + return fmt.Errorf("failed to apply preflight plugin: %w", err) + } _, err = d.coreClient.Pods(d.namespace).Create(ctx, pod, metav1.CreateOptions{}) - return err + if err != nil { + return fmt.Errorf("failed to create preflight pod: %w", err) + } + return nil } func (d *PodPreflightDeployer) Status(ctx context.Context) (corev1.PodStatus, error) { @@ -70,6 +96,11 @@ func (d *PodPreflightDeployer) Cleanup(ctx context.Context) error { if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete pod %s/%s: %w", d.namespace, preflightPodName, err) } + + err = d.coreClient.Secrets(d.namespace).Delete(ctx, preflightEncryptionConfigSecretName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete secret %s/%s: %w", d.namespace, preflightEncryptionConfigSecretName, err) + } return nil } diff --git a/pkg/operator/encryption/kms/preflight/deployer_test.go b/pkg/operator/encryption/kms/preflight/deployer_test.go index 3aeb2ef4d1..e60154a89c 100644 --- a/pkg/operator/encryption/kms/preflight/deployer_test.go +++ b/pkg/operator/encryption/kms/preflight/deployer_test.go @@ -6,12 +6,16 @@ import ( "testing" "time" + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/library-go/pkg/operator/encryption/encryptiondata" + encryptiontesting "github.com/openshift/library-go/pkg/operator/encryption/testing" "github.com/openshift/library-go/pkg/operator/resource/resourceread" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + apiserverconfigv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" "k8s.io/client-go/kubernetes/fake" clienttesting "k8s.io/client-go/testing" ) @@ -22,6 +26,7 @@ const ( testOperatorImage = "quay.io/openshift-release-dev/ocp-v5.0-art-dev@sha256:test" configHashAnnotationKey = "encryption.apiserver.operator.openshift.io/kms-preflight-config-hash" testDeployerTimeout = 10 * time.Second + testReferenceDataNS = "openshift-config" ) var testOperatorCommand = []string{"cluster-kube-apiserver-operator", "kms-preflight"} @@ -48,6 +53,39 @@ spec: - key: node-role.kubernetes.io/master operator: Exists effect: NoExecute + initContainers: + - name: vault-kms-plugin-1 + image: registry.example.com/kms-plugin@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890 + args: + - -listen-address=unix:///var/run/kmsplugin/kms.sock + - -vault-address=https://vault.example.com + - -transit-mount= + - -transit-key=test-transit-key + - -approle-role-id=test-role-id + - -approle-secret-id-path=/var/run/secrets/kms-plugin/kms-plugin-secret-vault-approle-secret_secret-id-1 + - -tls-ca-file=/var/run/secrets/kms-plugin/kms-plugin-configmap-vault-ca-bundle_ca-bundle.crt-1 + - -metrics-port=0 + imagePullPolicy: IfNotPresent + restartPolicy: Always + terminationMessagePolicy: FallbackToLogsOnError + resources: + requests: + memory: 64Mi + cpu: 10m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: kms-plugin-socket + mountPath: /var/run/kmsplugin + - name: kms-plugins-data + mountPath: /var/run/secrets/kms-plugin + readOnly: true containers: - name: kms-preflight-check image: quay.io/openshift-release-dev/ocp-v5.0-art-dev@sha256:test @@ -75,12 +113,86 @@ spec: volumes: - name: kms-plugin-socket emptyDir: {} + - name: kms-plugins-data + secret: + secretName: kms-preflight-encryption-config ` +func testReferenceDataObjects(t *testing.T) []runtime.Object { + t.Helper() + + return []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-approle-secret", + Namespace: testReferenceDataNS, + }, + Data: map[string][]byte{ + "role-id": []byte("test-role-id"), + "secret-id": []byte("test-secret-id"), + }, + }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-ca-bundle", + Namespace: testReferenceDataNS, + }, + Data: map[string]string{ + "ca-bundle.crt": "test-ca-cert", + }, + }, + } +} + +func testPreflightEncryptionConfig(t *testing.T) *encryptiondata.Config { + t.Helper() + + var secretData encryptiondata.KMSPluginsReferenceData + if err := secretData.SetFromRawKey("1", "vault-approle-secret_role-id", []byte("test-role-id")); err != nil { + t.Fatalf("failed to set secret reference data: %v", err) + } + if err := secretData.SetFromRawKey("1", "vault-approle-secret_secret-id", []byte("test-secret-id")); err != nil { + t.Fatalf("failed to set secret reference data: %v", err) + } + + var configMapData encryptiondata.KMSPluginsReferenceData + if err := configMapData.SetFromRawKey("1", "vault-ca-bundle_ca-bundle.crt", []byte("test-ca-cert")); err != nil { + t.Fatalf("failed to set configmap reference data: %v", err) + } + + return &encryptiondata.Config{ + Encryption: &apiserverconfigv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "EncryptionConfiguration", + APIVersion: "apiserver.config.k8s.io/v1", + }, + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1_secrets", + Endpoint: kmsSocketEndpoint, + Timeout: &metav1.Duration{Duration: testDeployerTimeout}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + KMSPlugins: map[string]configv1.KMSPluginConfig{ + "1": encryptiontesting.DefaultKMSPluginConfig, + }, + KMSPluginsSecretData: secretData, + KMSPluginsConfigMapData: configMapData, + } +} + func newTestDeployer(t *testing.T, objects ...runtime.Object) (*PodPreflightDeployer, *fake.Clientset) { t.Helper() - kubeClient := fake.NewSimpleClientset(objects...) + allObjects := append(testReferenceDataObjects(t), objects...) + kubeClient := fake.NewSimpleClientset(allObjects...) deployer := NewPodPreflightDeployer( testNamespace, kubeClient.CoreV1(), @@ -94,13 +206,14 @@ func newTestDeployer(t *testing.T, objects ...runtime.Object) (*PodPreflightDepl func TestPodPreflightDeployer_Deploy(t *testing.T) { ctx := context.Background() deployer, kubeClient := newTestDeployer(t) + encryptionConfig := testPreflightEncryptionConfig(t) expectedPod, err := resourceread.ReadPodV1([]byte(expectedDeployPodYAML)) if err != nil { t.Fatalf("failed to parse expected pod YAML: %v", err) } - if err := deployer.Deploy(ctx, testConfigHash); err != nil { + if err := deployer.Deploy(ctx, testConfigHash, encryptionConfig); err != nil { t.Fatalf("Deploy() error = %v", err) } @@ -129,6 +242,30 @@ func TestPodPreflightDeployer_Deploy(t *testing.T) { } } +func TestPodPreflightDeployer_Deploy_emptyConfigHash(t *testing.T) { + deployer, kubeClient := newTestDeployer(t) + + err := deployer.Deploy(context.Background(), "", testPreflightEncryptionConfig(t)) + if err == nil || !strings.Contains(err.Error(), "configHash is empty") { + t.Fatalf("expected configHash is empty error, got %v", err) + } + if len(kubeClient.Actions()) != 0 { + t.Fatalf("expected no client actions, got %d: %#v", len(kubeClient.Actions()), kubeClient.Actions()) + } +} + +func TestPodPreflightDeployer_Deploy_nilEncryptionConfig(t *testing.T) { + deployer, kubeClient := newTestDeployer(t) + + err := deployer.Deploy(context.Background(), testConfigHash, nil) + if err == nil || !strings.Contains(err.Error(), "encryptionConfig is nil") { + t.Fatalf("expected encryptionConfig is nil error, got %v", err) + } + if len(kubeClient.Actions()) != 0 { + t.Fatalf("expected no client actions, got %d: %#v", len(kubeClient.Actions()), kubeClient.Actions()) + } +} + func TestPodPreflightDeployer_Status(t *testing.T) { scenarios := []struct { name string @@ -198,8 +335,8 @@ func TestPodPreflightDeployer_Cleanup(t *testing.T) { name: "deletes existing pod", objects: []runtime.Object{existingPod}, verifyActions: func(t *testing.T, actions []clienttesting.Action) { - if len(actions) != 1 { - t.Fatalf("expected 1 action, got %d: %#v", len(actions), actions) + if len(actions) != 2 { + t.Fatalf("expected 2 actions, got %d: %#v", len(actions), actions) } if !actions[0].Matches("delete", "pods") { t.Fatalf("unexpected action: %#v", actions[0]) @@ -214,17 +351,23 @@ func TestPodPreflightDeployer_Cleanup(t *testing.T) { if deleteAction.GetNamespace() != testNamespace { t.Fatalf("unexpected namespace %q", deleteAction.GetNamespace()) } + if !actions[1].Matches("delete", "secrets") { + t.Fatalf("unexpected action: %#v", actions[1]) + } }, }, { name: "missing pod is not an error", verifyActions: func(t *testing.T, actions []clienttesting.Action) { - if len(actions) != 1 { - t.Fatalf("expected 1 action, got %d: %#v", len(actions), actions) + if len(actions) != 2 { + t.Fatalf("expected 2 actions, got %d: %#v", len(actions), actions) } if !actions[0].Matches("delete", "pods") { t.Fatalf("unexpected action: %#v", actions[0]) } + if !actions[1].Matches("delete", "secrets") { + t.Fatalf("unexpected action: %#v", actions[1]) + } }, }, { @@ -245,6 +388,26 @@ func TestPodPreflightDeployer_Cleanup(t *testing.T) { } }, }, + { + name: "delete secret error is returned", + setupClient: func(kubeClient *fake.Clientset) { + kubeClient.PrependReactor("delete", "secrets", func(action clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(corev1.Resource("secrets"), preflightEncryptionConfigSecretName, nil) + }) + }, + expectErr: "failed to delete secret", + verifyActions: func(t *testing.T, actions []clienttesting.Action) { + if len(actions) != 2 { + t.Fatalf("expected 2 actions, got %d: %#v", len(actions), actions) + } + if !actions[0].Matches("delete", "pods") { + t.Fatalf("unexpected action: %#v", actions[0]) + } + if !actions[1].Matches("delete", "secrets") { + t.Fatalf("unexpected action: %#v", actions[1]) + } + }, + }, } for _, scenario := range scenarios { diff --git a/pkg/operator/encryption/kms/preflight/pod_template_test.go b/pkg/operator/encryption/kms/preflight/pod_template_test.go index 5e720d8dee..3e1b519e09 100644 --- a/pkg/operator/encryption/kms/preflight/pod_template_test.go +++ b/pkg/operator/encryption/kms/preflight/pod_template_test.go @@ -51,12 +51,6 @@ spec: requests: memory: 50Mi cpu: 5m - volumeMounts: - - name: kms-plugin-socket - mountPath: /var/run/kmsplugin - volumes: - - name: kms-plugin-socket - emptyDir: {} ` func TestGeneratePodTemplate(t *testing.T) {