diff --git a/pkg/operator/encryption/controllers/key_controller.go b/pkg/operator/encryption/controllers/key_controller.go index d487e95a76..2b35acc95e 100644 --- a/pkg/operator/encryption/controllers/key_controller.go +++ b/pkg/operator/encryption/controllers/key_controller.go @@ -199,9 +199,18 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact // fills up the state with all resources and set identity write key if write key secrets // are missing. + var providerCfg kmsProviderConfig = noopKMSProviderConfig{} + if currentMode == state.KMS { + var err error + providerCfg, err = newKMSProviderConfig(apiEncryptionConfiguration.KMS) + if err != nil { + return err + } + } + var commonReason *string for gr, grKeys := range desiredEncryptionState { - latestKeyID, internalReason, needed := needsNewKey(grKeys, currentMode, externalReason, encryptedGRs) + latestKeyID, internalReason, needed := needsNewKey(grKeys, currentMode, externalReason, encryptedGRs, providerCfg) if !needed { continue } @@ -228,7 +237,7 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact sort.Sort(sort.StringSlice(reasons)) internalReason := strings.Join(reasons, ", ") - keySecret, err := c.generateKeySecret(ctx, newKeyID, currentMode, apiEncryptionConfiguration, internalReason, externalReason) + keySecret, err := c.generateKeySecret(ctx, newKeyID, currentMode, apiEncryptionConfiguration, providerCfg, internalReason, externalReason) if err != nil { return fmt.Errorf("failed to create key: %v", err) } @@ -265,7 +274,7 @@ func (c *keyController) validateExistingSecret(ctx context.Context, keySecret *c return nil // we made this key earlier } -func (c *keyController) generateKeySecret(ctx context.Context, keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, internalReason, externalReason string) (*corev1.Secret, error) { +func (c *keyController) generateKeySecret(ctx context.Context, keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, providerCfg kmsProviderConfig, internalReason, externalReason string) (*corev1.Secret, error) { bs := crypto.ModeToNewKeyFunc[currentMode]() ks := state.KeyState{ Key: apiserverv1.Key{ @@ -287,11 +296,6 @@ func (c *keyController) generateKeySecret(ctx context.Context, keyID uint64, cur Plugin: apiServerEncryption.KMS, } - providerCfg, err := newKMSProviderConfig(apiServerEncryption.KMS) - if err != nil { - return nil, err - } - if secretName, expectedKeys, err := providerCfg.referencedSecretName(); err != nil { return nil, err } else if len(secretName) > 0 { @@ -363,7 +367,7 @@ func (c *keyController) getCurrentModeReasonAndEncryptionConfig(ctx context.Cont // needsNewKey checks whether a new key must be created for the given resource. If true, it also returns the latest // used key ID and a reason string. -func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, externalReason string, encryptedGRs []schema.GroupResource) (uint64, string, bool) { +func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, externalReason string, encryptedGRs []schema.GroupResource, providerCfg kmsProviderConfig) (uint64, string, bool) { // we always need to have some encryption keys unless we are turned off if len(grKeys.ReadKeys) == 0 { return 0, "key-does-not-exist", currentMode != state.Identity @@ -408,13 +412,25 @@ func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, extern if currentMode == state.KMS { // We are here because Encryption Mode is not changed + // However, we need to create a new key if migration-triggering fields + // in the KMS provider configuration have changed. + if latestKey.KMS == nil { + // A KMS-mode key without KMS state indicates a corrupted key secret. + // Do not create a new key on corrupted data. + klog.Errorf("KMS-mode key %q has nil KMS state, possibly corrupted key secret; skipping new key creation", latestKey.Key.Name) + return 0, "", false + } + migrationNeeded, err := providerCfg.migrationRequired(latestKey.KMS.Plugin) + if err != nil { + klog.Errorf("Failed to check KMS migration: %v", err) + return 0, "", false + } + if migrationNeeded { + return latestKeyID, "kms-provider-changed", true + } - // For now in Tech Preview v1, we don't support configurational changes. Therefore, - // it is pointless comparing the secrets. - - // For KMS mode, we don't do time-based rotation. Therefore, we shortcut here - // KMS keys are rotated externally by the KMS system. - // Moreover, we don't trigger new key when external reason is changed. + // For KMS mode, we don't do time-based rotation. KMS keys are rotated + // externally by the KMS provider. Moreover, we don't trigger new key when external reason is changed. // Because it would lead to duplicate providers which is not allowed. return 0, "", false } @@ -442,7 +458,21 @@ type kmsProviderConfig interface { // config and the specific data keys to carry from that configmap. Only the listed keys // are copied into the Key Secret; any other data in the referenced configmap is ignored. referencedConfigMapName() (string, []string, error) + // migrationRequired reports whether switching from latest (stored in + // the key secret) to this provider config requires a new encryption key. + migrationRequired(latest configv1.KMSPluginConfig) (bool, error) +} + +// noopKMSProviderConfig is a safe zero-value implementation used for non-KMS modes. +// All methods return empty/false so callers never need nil checks. +type noopKMSProviderConfig struct{} + +func (noopKMSProviderConfig) referencedSecretName() (string, []string, error) { return "", nil, nil } +func (noopKMSProviderConfig) referencedConfigMapName() (string, []string, error) { return "", nil, nil } +func (noopKMSProviderConfig) migrationRequired(configv1.KMSPluginConfig) (bool, error) { + return false, fmt.Errorf("migrationRequired called on non-KMS provider") } +func (noopKMSProviderConfig) sourceConfig() interface{} { return nil } func newKMSProviderConfig(plugin configv1.KMSPluginConfig) (kmsProviderConfig, error) { switch plugin.Type { @@ -479,6 +509,30 @@ func (v *vaultProviderConfig) referencedConfigMapName() (string, []string, error return v.vault.TLS.CABundle.Name, []string{"ca-bundle.crt"}, nil } +func (v *vaultProviderConfig) migrationRequired(latest configv1.KMSPluginConfig) (bool, error) { + if latest.Type != configv1.VaultKMSProvider { + klog.V(2).Infof("KMS migration required: provider type changed from %q to %q", latest.Type, configv1.VaultKMSProvider) + return true, nil + } + if v.vault.VaultAddress != latest.Vault.VaultAddress { + klog.V(2).Infof("KMS migration required: VaultAddress changed from %q to %q", latest.Vault.VaultAddress, v.vault.VaultAddress) + return true, nil + } + if v.vault.VaultNamespace != latest.Vault.VaultNamespace { + klog.V(2).Infof("KMS migration required: VaultNamespace changed from %q to %q", latest.Vault.VaultNamespace, v.vault.VaultNamespace) + return true, nil + } + if v.vault.TransitMount != latest.Vault.TransitMount { + klog.V(2).Infof("KMS migration required: TransitMount changed from %q to %q", latest.Vault.TransitMount, v.vault.TransitMount) + return true, nil + } + if v.vault.TransitKey != latest.Vault.TransitKey { + klog.V(2).Infof("KMS migration required: TransitKey changed from %q to %q", latest.Vault.TransitKey, v.vault.TransitKey) + return true, nil + } + return false, nil +} + // TODO make this un-settable once set // ex: we could require the tech preview no upgrade flag to be set before we will honor this field type unsupportedEncryptionConfig struct { diff --git a/pkg/operator/encryption/controllers/key_controller_test.go b/pkg/operator/encryption/controllers/key_controller_test.go index 7c0bbf87f2..4f56d8f33e 100644 --- a/pkg/operator/encryption/controllers/key_controller_test.go +++ b/pkg/operator/encryption/controllers/key_controller_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" clocktesting "k8s.io/utils/clock/testing" corev1 "k8s.io/api/core/v1" @@ -768,6 +769,164 @@ func TestKeyController(t *testing.T) { } }, }, + + { + name: "creates a new KMS key when VaultAddress changes", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateMigratedEncryptionKeySecretWithKMSPluginConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, time.Now()), + encryptiontesting.CreateVaultAppRoleSecret("vault-approle-secret", "test-role-id", "test-secret-id"), + encryptiontesting.CreateVaultCABundleConfigMap("vault-ca-bundle", "test-ca-cert"), + }, + apiServerObjects: []runtime.Object{func() runtime.Object { + s := simpleAPIServer.DeepCopy() + changedConfig := encryptiontesting.DefaultKMSPluginConfig.DeepCopy() + changedConfig.Vault.VaultAddress = "https://vault-new.example.com" + s.Spec.Encryption = configv1.APIServerEncryption{Type: "KMS", KMS: *changedConfig} + return s + }()}, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "get:configmaps:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"}, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { + for _, action := range actions { + if action.Matches("create", "secrets") { + createAction := action.(clientgotesting.CreateAction) + actualSecret := createAction.GetObject().(*corev1.Secret) + + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"] != "KMS" { + ts.Errorf("expected mode KMS, got %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"]) + } + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"] != "secrets-kms-provider-changed" { + ts.Errorf("unexpected internal reason: %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"]) + } + if actualSecret.Name != "encryption-key-kms-6" { + ts.Errorf("expected key ID 6, got %s", actualSecret.Name) + } + + kmsProviderConfigData := actualSecret.Data["encryption.apiserver.operator.openshift.io-kms-plugin-config"] + providerConfig, err := encoding.DecodeKMSPluginConfig(kmsProviderConfigData) + if err != nil { + ts.Fatalf("failed to encode KMS config: %v", err) + } + if providerConfig.Vault.VaultAddress != "https://vault-new.example.com" { + ts.Errorf("expected new VaultAddress, got %s", providerConfig.Vault.VaultAddress) + } + return + } + } + ts.Errorf("the secret wasn't created") + }, + }, + + { + name: "creates a new KMS key when TransitKey changes", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateMigratedEncryptionKeySecretWithKMSPluginConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, time.Now()), + encryptiontesting.CreateVaultAppRoleSecret("vault-approle-secret", "test-role-id", "test-secret-id"), + encryptiontesting.CreateVaultCABundleConfigMap("vault-ca-bundle", "test-ca-cert"), + }, + apiServerObjects: []runtime.Object{func() runtime.Object { + s := simpleAPIServer.DeepCopy() + changedConfig := encryptiontesting.DefaultKMSPluginConfig.DeepCopy() + changedConfig.Vault.TransitKey = "new-transit-key" + s.Spec.Encryption = configv1.APIServerEncryption{Type: "KMS", KMS: *changedConfig} + return s + }()}, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "get:configmaps:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"}, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { + for _, action := range actions { + if action.Matches("create", "secrets") { + createAction := action.(clientgotesting.CreateAction) + actualSecret := createAction.GetObject().(*corev1.Secret) + + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"] != "secrets-kms-provider-changed" { + ts.Errorf("unexpected internal reason: %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"]) + } + + kmsProviderConfigData := actualSecret.Data["encryption.apiserver.operator.openshift.io-kms-plugin-config"] + providerConfig, err := encoding.DecodeKMSPluginConfig(kmsProviderConfigData) + if err != nil { + ts.Fatalf("failed to encode KMS config: %v", err) + } + if providerConfig.Vault.TransitKey != "new-transit-key" { + ts.Errorf("expected new TransitKey, got %s", providerConfig.Vault.TransitKey) + } + return + } + } + ts.Errorf("the secret wasn't created") + }, + }, + + { + name: "no-op when only KMSPluginImage changes (non-migration field)", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateMigratedEncryptionKeySecretWithKMSPluginConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, time.Now()), + }, + apiServerObjects: []runtime.Object{func() runtime.Object { + s := simpleAPIServer.DeepCopy() + changedConfig := encryptiontesting.DefaultKMSPluginConfig.DeepCopy() + changedConfig.Vault.KMSPluginImage = "registry.example.com/kms-plugin@sha256:0000000000000000000000000000000000000000000000000000000000000000" + s.Spec.Encryption = configv1.APIServerEncryption{Type: "KMS", KMS: *changedConfig} + return s + }()}, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed"}, + }, + + { + name: "no-op when only Authentication changes (non-migration field)", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateMigratedEncryptionKeySecretWithKMSPluginConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, time.Now()), + }, + apiServerObjects: []runtime.Object{func() runtime.Object { + s := simpleAPIServer.DeepCopy() + changedConfig := encryptiontesting.DefaultKMSPluginConfig.DeepCopy() + changedConfig.Vault.Authentication.AppRole.Secret.Name = "new-approle-secret" + s.Spec.Encryption = configv1.APIServerEncryption{Type: "KMS", KMS: *changedConfig} + return s + }()}, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed"}, + }, + + { + name: "no-op when only TLS changes (non-migration field)", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateMigratedEncryptionKeySecretWithKMSPluginConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, time.Now()), + }, + apiServerObjects: []runtime.Object{func() runtime.Object { + s := simpleAPIServer.DeepCopy() + changedConfig := encryptiontesting.DefaultKMSPluginConfig.DeepCopy() + changedConfig.Vault.TLS = configv1.VaultTLSConfig{ + CABundle: configv1.VaultConfigMapReference{Name: "my-ca"}, + } + s.Spec.Encryption = configv1.APIServerEncryption{Type: "KMS", KMS: *changedConfig} + return s + }()}, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed"}, + }, } for _, scenario := range scenarios { @@ -1140,3 +1299,118 @@ func TestGetCurrentModeReasonAndEncryptionConfig(t *testing.T) { }) } } + +func TestNeedsNewKey(t *testing.T) { + baseConfig := &configv1.KMSPluginConfig{ + Type: configv1.VaultKMSProvider, + Vault: configv1.VaultKMSPluginConfig{ + KMSPluginImage: "registry.example.com/kms-plugin@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + VaultAddress: "https://vault.example.com", + VaultNamespace: "ns1", + TransitMount: "transit", + TransitKey: "my-key", + Authentication: configv1.VaultAuthentication{ + Type: configv1.VaultAuthenticationTypeAppRole, + AppRole: configv1.VaultAppRoleAuthentication{ + Secret: configv1.VaultSecretReference{Name: "vault-approle-secret"}, + }, + }, + }, + } + + tests := []struct { + name string + latest *configv1.KMSPluginConfig + current *configv1.KMSPluginConfig + expected bool + }{ + { + name: "identical configs", + latest: baseConfig.DeepCopy(), + current: baseConfig.DeepCopy(), + expected: false, + }, + { + name: "different VaultAddress", + latest: baseConfig.DeepCopy(), + current: func() *configv1.KMSPluginConfig { + c := baseConfig.DeepCopy() + c.Vault.VaultAddress = "https://vault-new.example.com" + return c + }(), + expected: true, + }, + { + name: "different VaultNamespace", + latest: baseConfig.DeepCopy(), + current: func() *configv1.KMSPluginConfig { + c := baseConfig.DeepCopy() + c.Vault.VaultNamespace = "ns2" + return c + }(), + expected: true, + }, + { + name: "different TransitKey", + latest: baseConfig.DeepCopy(), + current: func() *configv1.KMSPluginConfig { + c := baseConfig.DeepCopy() + c.Vault.TransitKey = "new-key" + return c + }(), + expected: true, + }, + { + name: "different TransitMount", + latest: baseConfig.DeepCopy(), + current: func() *configv1.KMSPluginConfig { + c := baseConfig.DeepCopy() + c.Vault.TransitMount = "custom-transit" + return c + }(), + expected: true, + }, + { + name: "different KMSPluginImage only", + latest: baseConfig.DeepCopy(), + current: func() *configv1.KMSPluginConfig { + c := baseConfig.DeepCopy() + c.Vault.KMSPluginImage = "registry.example.com/kms-plugin@sha256:0000000000000000000000000000000000000000000000000000000000000000" + return c + }(), + expected: false, + }, + { + name: "different TLS only", + latest: baseConfig.DeepCopy(), + current: func() *configv1.KMSPluginConfig { + c := baseConfig.DeepCopy() + c.Vault.TLS = configv1.VaultTLSConfig{ + CABundle: configv1.VaultConfigMapReference{Name: "my-ca"}, + } + return c + }(), + expected: false, + }, + { + name: "different Authentication only", + latest: baseConfig.DeepCopy(), + current: func() *configv1.KMSPluginConfig { + c := baseConfig.DeepCopy() + c.Vault.Authentication.AppRole.Secret.Name = "new-secret" + return c + }(), + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + providerCfg, err := newKMSProviderConfig(*tt.current) + require.NoError(t, err) + got, err := providerCfg.migrationRequired(*tt.latest) + require.NoError(t, err) + require.Equal(t, tt.expected, got) + }) + } +} diff --git a/test/e2e-encryption/encryption_test.go b/test/e2e-encryption/encryption_test.go index f659f51489..e05d6fe307 100644 --- a/test/e2e-encryption/encryption_test.go +++ b/test/e2e-encryption/encryption_test.go @@ -634,33 +634,64 @@ func TestEncryptionIntegration(tt *testing.T) { verifyKMSSecretData() verifyKMSConfigMapData() + t.Logf("KMS-to-KMS migration: change VaultAddress") + _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"kms":{"vault":{"vaultAddress":"https://vault-new.example.com"}}}}}`), metav1.PatchOptions{}) + require.NoError(t, err) + waitForKeys(12) + kms13 := kmsPluginName("kubeapiservers", "13") + kms13Sched := kmsPluginName("kubeschedulers", "13") + waitForConfigs( + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,kms:%s,aescbc:11,identity;kubeschedulers.operator.openshift.io=kms:%s,kms:%s,aescbc:11,identity", kms12, kms13, kms12Sched, kms13Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,kms:%s,aescbc:11,identity;kubeschedulers.operator.openshift.io=kms:%s,kms:%s,aescbc:11,identity", kms13, kms12, kms13Sched, kms12Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,kms:%s,identity;kubeschedulers.operator.openshift.io=kms:%s,kms:%s,identity", kms13, kms12, kms13Sched, kms12Sched), + ) + waitForMigration("13") + waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + + t.Logf("Verify KMS-to-KMS migration key secret contains updated plugin config") + kmsKeySecret13, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-key-%s-13", component), metav1.GetOptions{}) + require.NoError(t, err) + kmsPluginConfigData13 := kmsKeySecret13.Data["encryption.apiserver.operator.openshift.io-kms-plugin-config"] + require.NotEmpty(t, kmsPluginConfigData13) + pluginConfig13, err := encoding.DecodeKMSPluginConfig(kmsPluginConfigData13) + require.NoError(t, err) + require.Equal(t, "https://vault-new.example.com", pluginConfig13.Vault.VaultAddress) + require.Equal(t, "test-transit-key", pluginConfig13.Vault.TransitKey) + + t.Logf("KMS non-migration change: only KMSPluginImage changes (no new key expected)") + _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"KMS","kms":{"type":"Vault","vault":{"kmsPluginImage":"registry.example.com/kms-plugin@sha256:0000000000000000000000000000000000000000000000000000000000000000","vaultAddress":"https://vault-new.example.com","authentication":{"type":"AppRole","appRole":{"secret":{"name":"vault-approle-secret"}}},"transitKey":"test-transit-key"}}}}}`), metav1.PatchOptions{}) + require.NoError(t, err) + time.Sleep(5 * time.Second) + waitForKeys(12) + waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + t.Logf("Delete the encryption-config while in KMS mode") _, err = kubeClient.CoreV1().Secrets("openshift-config-managed").Patch(ctx, fmt.Sprintf("encryption-config-%s", component), types.JSONPatchType, []byte(`[{"op":"remove","path":"/metadata/finalizers"}]`), metav1.PatchOptions{}) require.NoError(t, err) err = kubeClient.CoreV1().Secrets("openshift-config-managed").Delete(ctx, fmt.Sprintf("encryption-config-%s", component), metav1.DeleteOptions{}) require.NoError(t, err) waitForConfigs( - fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:11,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:11,identity", kms12, kms12Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,kms:%s,identity;kubeschedulers.operator.openshift.io=kms:%s,kms:%s,identity", kms13, kms12, kms13Sched, kms12Sched), ) waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) t.Logf("Delete the operand config while in KMS mode") deployer.DeleteOperandConfig() waitForConfigs( - // kms12 is migrated and hence only one needed, but we rotate through identity - fmt.Sprintf("kubeapiservers.operator.openshift.io=identity,kms:%s,aescbc:11;kubeschedulers.operator.openshift.io=identity,kms:%s,aescbc:11", kms12, kms12Sched), - // kms12 is migrated, plus one backed key (11) - fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:11,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:11,identity", kms12, kms12Sched), + // kms13 is migrated and hence only one needed, but we rotate through identity + fmt.Sprintf("kubeapiservers.operator.openshift.io=identity,kms:%s,kms:%s;kubeschedulers.operator.openshift.io=identity,kms:%s,kms:%s", kms13, kms12, kms13Sched, kms12Sched), + // kms13 is migrated, plus one backed key (12) + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,kms:%s,identity;kubeschedulers.operator.openshift.io=kms:%s,kms:%s,identity", kms13, kms12, kms13Sched, kms12Sched), ) waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) t.Logf("Switch to identity from KMS") _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"identity","kms":null}}}`), metav1.PatchOptions{}) require.NoError(t, err) - waitForKeys(12) + waitForKeys(13) waitForConfigs( - fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:11,identity,aesgcm:13;kubeschedulers.operator.openshift.io=kms:%s,aescbc:11,identity,aesgcm:13", kms12, kms12Sched), - fmt.Sprintf("kubeapiservers.operator.openshift.io=identity,kms:%s,aescbc:11,aesgcm:13;kubeschedulers.operator.openshift.io=identity,kms:%s,aescbc:11,aesgcm:13", kms12, kms12Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,kms:%s,identity,aesgcm:14;kubeschedulers.operator.openshift.io=kms:%s,kms:%s,identity,aesgcm:14", kms13, kms12, kms13Sched, kms12Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=identity,kms:%s,kms:%s,aesgcm:14;kubeschedulers.operator.openshift.io=identity,kms:%s,kms:%s,aesgcm:14", kms13, kms12, kms13Sched, kms12Sched), ) waitForConditionStatus("Encrypted", operatorv1.ConditionFalse) }