Skip to content

Commit f5987de

Browse files
ardagucluardaguclu
committed
Inject health reporter as sidecar container
1 parent b0f5231 commit f5987de

5 files changed

Lines changed: 267 additions & 6 deletions

File tree

‎pkg/operator/encryption/kms/pluginlifecycle/builder.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@ type KMSPluginBuilder struct {
2020
encryptionConfig *encryptiondata.Config
2121
encryptionConfigSecretName string
2222
staticPod bool
23+
24+
enableHealthReporter bool
25+
healthReporterContainerName string
26+
healthReporterOperatorCmd string
27+
healthReporterImage string
2328
}
2429

2530
// NewKMSPluginBuilder creates a builder that defaults to deployment mode.
@@ -44,6 +49,17 @@ func (b *KMSPluginBuilder) AsStaticPod() *KMSPluginBuilder {
4449
return b
4550
}
4651

52+
// WithHealthReporter enables injection of a health-reporter sidecar.
53+
// containerName is used as both the container name and the subcommand name.
54+
// operatorCmd is the parent binary (e.g. "cluster-kube-apiserver-operator").
55+
func (b *KMSPluginBuilder) WithHealthReporter(containerName, operatorCmd, operatorImage string) *KMSPluginBuilder {
56+
b.enableHealthReporter = true
57+
b.healthReporterContainerName = containerName
58+
b.healthReporterOperatorCmd = operatorCmd
59+
b.healthReporterImage = operatorImage
60+
return b
61+
}
62+
4763
// Apply mutates the given pod spec by injecting KMS plugin sidecars, volumes,
4864
// and volume mounts. containerName identifies the API server container that
4965
// needs the socket volume mount.
@@ -83,9 +99,11 @@ func (b *KMSPluginBuilder) Apply(podSpec *corev1.PodSpec, containerName string)
8399
socketVolumeMount := corev1.VolumeMount{Name: kmsPluginSocketVolumeName, MountPath: kmsPluginSocketMountPath, ReadOnly: false}
84100
refDataVolumeMount := corev1.VolumeMount{Name: refDataVolumeName, MountPath: refDataMountPath, ReadOnly: true}
85101

102+
sockets := make([]string, 0, len(kmsConfigurations))
86103
for _, kmsConfiguration := range kmsConfigurations {
87104
// ExtractUniqueAndSortedKMSConfigurations function rewrites the .Name field to include only the key ID
88105
keyID := kmsConfiguration.Name
106+
sockets = append(sockets, kmsConfiguration.Endpoint)
89107

90108
pluginConfig, ok := b.encryptionConfig.KMSPlugins[keyID]
91109
if !ok {
@@ -137,6 +155,12 @@ func (b *KMSPluginBuilder) Apply(podSpec *corev1.PodSpec, containerName string)
137155
}
138156
}
139157

158+
if b.enableHealthReporter {
159+
if err := b.applyHealthReporter(podSpec, sockets); err != nil {
160+
return err
161+
}
162+
}
163+
140164
return nil
141165
}
142166

‎pkg/operator/encryption/kms/pluginlifecycle/builder_test.go‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,101 @@ func TestKMSPluginBuilder_Apply(t *testing.T) {
123123
}
124124
}
125125

126+
func TestKMSPluginBuilder_WithHealthReporter(t *testing.T) {
127+
f := newSidecarTestFixtures(t)
128+
cfg, err := encryptiondata.FromSecret(f.encryptionConfigSecret)
129+
require.NoError(t, err)
130+
131+
t.Run("health reporter injected with correct args and socket mount", func(t *testing.T) {
132+
podSpec := &corev1.PodSpec{
133+
Containers: []corev1.Container{{Name: "kube-apiserver"}},
134+
Volumes: []corev1.Volume{f.resourceDirVolume},
135+
}
136+
err := NewKMSPluginBuilder().
137+
FromEncryptionConfig("encryption-config", cfg).
138+
WithHealthReporter("kms-health-reporter", "cluster-kube-apiserver-operator", "quay.io/test/operator:latest").
139+
Apply(podSpec, "kube-apiserver")
140+
require.NoError(t, err)
141+
142+
var reporter *corev1.Container
143+
for i := range podSpec.InitContainers {
144+
if podSpec.InitContainers[i].Name == "kms-health-reporter" {
145+
reporter = &podSpec.InitContainers[i]
146+
break
147+
}
148+
}
149+
require.NotNil(t, reporter, "health reporter container must be injected")
150+
require.Equal(t, "quay.io/test/operator:latest", reporter.Image)
151+
require.Equal(t, []string{"cluster-kube-apiserver-operator", "kms-health-reporter"}, reporter.Command)
152+
require.Contains(t, reporter.Args, "--kms-sockets=unix:///var/run/kmsplugin/kms-555.sock")
153+
require.Contains(t, reporter.Args, "--node-name=$(NODE_NAME)")
154+
155+
hasSocketMount := false
156+
for _, m := range reporter.VolumeMounts {
157+
if m.Name == "kms-plugin-socket" {
158+
hasSocketMount = true
159+
require.True(t, m.ReadOnly)
160+
}
161+
}
162+
require.True(t, hasSocketMount)
163+
164+
require.Len(t, reporter.Env, 1)
165+
require.Equal(t, "NODE_NAME", reporter.Env[0].Name)
166+
require.Equal(t, "spec.nodeName", reporter.Env[0].ValueFrom.FieldRef.FieldPath)
167+
})
168+
169+
t.Run("static pod mode: health reporter gets root UID", func(t *testing.T) {
170+
podSpec := &corev1.PodSpec{
171+
Containers: []corev1.Container{{Name: "kube-apiserver"}},
172+
Volumes: []corev1.Volume{f.resourceDirVolume},
173+
}
174+
err := NewKMSPluginBuilder().
175+
FromEncryptionConfig("encryption-config", cfg).
176+
AsStaticPod().
177+
WithHealthReporter("kms-health-reporter", "cluster-kube-apiserver-operator", "quay.io/test/operator:latest").
178+
Apply(podSpec, "kube-apiserver")
179+
require.NoError(t, err)
180+
181+
var reporter *corev1.Container
182+
for i := range podSpec.InitContainers {
183+
if podSpec.InitContainers[i].Name == "kms-health-reporter" {
184+
reporter = &podSpec.InitContainers[i]
185+
break
186+
}
187+
}
188+
require.NotNil(t, reporter)
189+
require.Equal(t, ptr.To(int64(0)), reporter.SecurityContext.RunAsUser)
190+
require.Contains(t, reporter.Args, "--kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig")
191+
})
192+
193+
t.Run("without WithHealthReporter: no health reporter", func(t *testing.T) {
194+
podSpec := &corev1.PodSpec{
195+
Containers: []corev1.Container{{Name: "kube-apiserver"}},
196+
Volumes: []corev1.Volume{f.resourceDirVolume},
197+
}
198+
err := NewKMSPluginBuilder().
199+
FromEncryptionConfig("encryption-config", cfg).
200+
Apply(podSpec, "kube-apiserver")
201+
require.NoError(t, err)
202+
203+
for _, c := range podSpec.InitContainers {
204+
require.NotEqual(t, "kms-health-reporter", c.Name, "health reporter must not be injected without WithHealthReporter")
205+
}
206+
})
207+
208+
t.Run("empty image: returns error", func(t *testing.T) {
209+
podSpec := &corev1.PodSpec{
210+
Containers: []corev1.Container{{Name: "kube-apiserver"}},
211+
Volumes: []corev1.Volume{f.resourceDirVolume},
212+
}
213+
err := NewKMSPluginBuilder().
214+
FromEncryptionConfig("encryption-config", cfg).
215+
WithHealthReporter("kms-health-reporter", "cluster-kube-apiserver-operator", "").
216+
Apply(podSpec, "kube-apiserver")
217+
require.ErrorContains(t, err, "health reporter image is required")
218+
})
219+
}
220+
126221
func TestKMSPluginBuilder_OrderIndependence(t *testing.T) {
127222
f := newSidecarTestFixtures(t)
128223
cfg, err := encryptiondata.FromSecret(f.encryptionConfigSecret)

‎pkg/operator/encryption/kms/pluginlifecycle/health_reporter.go‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package pluginlifecycle
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
corev1 "k8s.io/api/core/v1"
8+
"k8s.io/apimachinery/pkg/api/resource"
9+
"k8s.io/utils/ptr"
10+
)
11+
12+
// defaultStaticPodKubeconfig is reused from the cert-syncer kubeconfig because
13+
// in-cluster config does not work on host-network static pods (kubernetes.default.svc
14+
// does not resolve). This matches the pattern used by the startup monitor.
15+
// TODO: move to a dedicated least-privilege kubeconfig once available.
16+
const defaultStaticPodKubeconfig = "/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig"
17+
18+
func (b *KMSPluginBuilder) applyHealthReporter(podSpec *corev1.PodSpec, sockets []string) error {
19+
if b.healthReporterImage == "" {
20+
return fmt.Errorf("health reporter image is required when WithHealthReporter is used")
21+
}
22+
23+
var kubeconfig string
24+
if b.staticPod {
25+
kubeconfig = defaultStaticPodKubeconfig
26+
}
27+
28+
reporter := &healthReporter{name: b.healthReporterContainerName, operatorCmd: b.healthReporterOperatorCmd, image: b.healthReporterImage, sockets: sockets, kubeconfig: kubeconfig}
29+
if err := ensureSidecarContainer(podSpec, reporter); err != nil {
30+
return err
31+
}
32+
33+
socketMount := corev1.VolumeMount{Name: kmsPluginSocketVolumeName, MountPath: kmsPluginSocketMountPath, ReadOnly: true}
34+
if err := ensureVolumeMountInContainer(podSpec.InitContainers, reporter.Name(), socketMount); err != nil {
35+
return err
36+
}
37+
38+
if b.staticPod {
39+
resourceDirMount := corev1.VolumeMount{Name: resourceDirVolumeName, MountPath: resourcesDir, ReadOnly: true}
40+
if err := ensureVolumeMountInContainer(podSpec.InitContainers, reporter.Name(), resourceDirMount); err != nil {
41+
return err
42+
}
43+
if err := setRunAsRoot(podSpec.InitContainers, reporter.Name()); err != nil {
44+
return err
45+
}
46+
}
47+
48+
return nil
49+
}
50+
51+
type healthReporter struct {
52+
name string
53+
operatorCmd string
54+
image string
55+
sockets []string
56+
kubeconfig string
57+
}
58+
59+
func (h *healthReporter) Name() string {
60+
return h.name
61+
}
62+
63+
func (h *healthReporter) BuildSidecarContainer() (corev1.Container, error) {
64+
args := []string{
65+
fmt.Sprintf("--kms-sockets=%s", strings.Join(h.sockets, ",")),
66+
"--node-name=$(NODE_NAME)",
67+
}
68+
if h.kubeconfig != "" {
69+
args = append(args, fmt.Sprintf("--kubeconfig=%s", h.kubeconfig))
70+
}
71+
72+
return corev1.Container{
73+
Name: h.name,
74+
Image: h.image,
75+
Command: []string{h.operatorCmd, h.name},
76+
Args: args,
77+
ImagePullPolicy: corev1.PullIfNotPresent,
78+
RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways),
79+
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
80+
Env: []corev1.EnvVar{
81+
{
82+
Name: "NODE_NAME",
83+
ValueFrom: &corev1.EnvVarSource{
84+
FieldRef: &corev1.ObjectFieldSelector{
85+
FieldPath: "spec.nodeName",
86+
},
87+
},
88+
},
89+
},
90+
Resources: corev1.ResourceRequirements{
91+
Requests: corev1.ResourceList{
92+
corev1.ResourceMemory: resource.MustParse("32Mi"),
93+
corev1.ResourceCPU: resource.MustParse("10m"),
94+
},
95+
},
96+
SecurityContext: &corev1.SecurityContext{
97+
ReadOnlyRootFilesystem: ptr.To(true),
98+
AllowPrivilegeEscalation: ptr.To(false),
99+
Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}},
100+
SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault},
101+
},
102+
}, nil
103+
}

‎pkg/operator/encryption/kms/pluginlifecycle/sidecar.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@ const (
2121
)
2222

2323
const (
24-
kmsPluginSocketVolumeName = "kms-plugin-socket"
25-
kmsPluginSocketMountPath = "/var/run/kmsplugin"
24+
kmsPluginSocketVolumeName = "kms-plugin-socket"
25+
kmsPluginSocketMountPath = "/var/run/kmsplugin"
26+
healthReporterContainerName = "kms-health-reporter"
2627
)
2728

2829
// sidecarProvider abstracts the construction of a KMS plugin sidecar container for a specific provider (e.g. Vault).
@@ -52,7 +53,7 @@ func newSidecarProvider(keyID string, udsPath string, pluginConfig configv1.KMSP
5253
//
5354
// It is a no-op when the KMSEncryption feature gate is not enabled or the encryption-config secret does not exist.
5455
// The secretClient should be uncached to avoid injecting sidecars based on a stale encryption configuration.
55-
func AddKMSPluginSidecarToStaticPodSpec(ctx context.Context, podSpec *corev1.PodSpec, containerName string, encryptionConfigNamespace string, encryptionConfigSecretName string, secretClient corev1client.SecretsGetter, featureGateAccessor featuregates.FeatureGateAccess) error {
56+
func AddKMSPluginSidecarToStaticPodSpec(ctx context.Context, podSpec *corev1.PodSpec, containerName string, encryptionConfigNamespace string, encryptionConfigSecretName string, secretClient corev1client.SecretsGetter, featureGateAccessor featuregates.FeatureGateAccess, operatorCmd string, operatorImage string) error {
5657
if !featureGateAccessor.AreInitialFeatureGatesObserved() {
5758
return nil
5859
}
@@ -75,14 +76,15 @@ func AddKMSPluginSidecarToStaticPodSpec(ctx context.Context, podSpec *corev1.Pod
7576
return NewKMSPluginBuilder().
7677
FromEncryptionConfig(encryptionConfigSecretName, cfg).
7778
AsStaticPod().
79+
WithHealthReporter(healthReporterContainerName, operatorCmd, operatorImage).
7880
Apply(podSpec, containerName)
7981
}
8082

8183
// AddKMSPluginSidecarToPodSpec injects KMS plugin sidecar containers into an aggregated API server pod spec (e.g., openshift-apiserver, oauth-apiserver).
8284
//
8385
// It is a no-op when the KMSEncryption feature gate is not enabled or the encryption-config secret does not exist.
8486
// The secretClient should be uncached to avoid injecting sidecars based on a stale encryption configuration.
85-
func AddKMSPluginSidecarToPodSpec(ctx context.Context, podSpec *corev1.PodSpec, containerName string, encryptionConfigNamespace string, encryptionConfigSecretName string, secretClient corev1client.SecretsGetter, featureGateAccessor featuregates.FeatureGateAccess) error {
87+
func AddKMSPluginSidecarToPodSpec(ctx context.Context, podSpec *corev1.PodSpec, containerName string, encryptionConfigNamespace string, encryptionConfigSecretName string, secretClient corev1client.SecretsGetter, featureGateAccessor featuregates.FeatureGateAccess, operatorCmd string, operatorImage string) error {
8688
if !featureGateAccessor.AreInitialFeatureGatesObserved() {
8789
return nil
8890
}
@@ -104,6 +106,7 @@ func AddKMSPluginSidecarToPodSpec(ctx context.Context, podSpec *corev1.PodSpec,
104106

105107
return NewKMSPluginBuilder().
106108
FromEncryptionConfig(encryptionConfigSecretName, cfg).
109+
WithHealthReporter(healthReporterContainerName, operatorCmd, operatorImage).
107110
Apply(podSpec, containerName)
108111
}
109112

‎pkg/operator/encryption/kms/pluginlifecycle/sidecar_test.go‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,39 @@ func TestAddKMSPluginSidecarToPodSpec(t *testing.T) {
126126
"-metrics-port=0",
127127
}
128128

129+
expectedHealthReporter := func(sockets string) corev1.Container {
130+
return corev1.Container{
131+
Name: "kms-health-reporter",
132+
Image: "quay.io/test/operator:latest",
133+
Command: []string{"cluster-kube-apiserver-operator", "kms-health-reporter"},
134+
Args: []string{fmt.Sprintf("--kms-sockets=%s", sockets), "--node-name=$(NODE_NAME)"},
135+
ImagePullPolicy: corev1.PullIfNotPresent,
136+
RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways),
137+
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
138+
Env: []corev1.EnvVar{
139+
{
140+
Name: "NODE_NAME",
141+
ValueFrom: &corev1.EnvVarSource{
142+
FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"},
143+
},
144+
},
145+
},
146+
Resources: corev1.ResourceRequirements{
147+
Requests: corev1.ResourceList{
148+
corev1.ResourceMemory: resource.MustParse("32Mi"),
149+
corev1.ResourceCPU: resource.MustParse("10m"),
150+
},
151+
},
152+
SecurityContext: &corev1.SecurityContext{
153+
ReadOnlyRootFilesystem: ptr.To(true),
154+
AllowPrivilegeEscalation: ptr.To(false),
155+
Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}},
156+
SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault},
157+
},
158+
VolumeMounts: []corev1.VolumeMount{{Name: "kms-plugin-socket", MountPath: "/var/run/kmsplugin", ReadOnly: true}},
159+
}
160+
}
161+
129162
socketMount := corev1.VolumeMount{
130163
Name: "kms-plugin-socket",
131164
MountPath: "/var/run/kmsplugin",
@@ -194,6 +227,7 @@ func TestAddKMSPluginSidecarToPodSpec(t *testing.T) {
194227
},
195228
VolumeMounts: []corev1.VolumeMount{socketMount, refDataMount},
196229
},
230+
expectedHealthReporter("unix:///var/run/kmsplugin/kms-555.sock"),
197231
},
198232
Volumes: []corev1.Volume{f.resourceDirVolume, socketVolume, refDataVolume},
199233
},
@@ -279,6 +313,7 @@ func TestAddKMSPluginSidecarToPodSpec(t *testing.T) {
279313
},
280314
VolumeMounts: []corev1.VolumeMount{socketMount, refDataMount},
281315
},
316+
expectedHealthReporter("unix:///var/run/kmsplugin/kms-777.sock,unix:///var/run/kmsplugin/kms-555.sock"),
282317
},
283318
Volumes: []corev1.Volume{f.resourceDirVolume, socketVolume, refDataVolume},
284319
},
@@ -513,6 +548,7 @@ func TestAddKMSPluginSidecarToPodSpec(t *testing.T) {
513548
},
514549
VolumeMounts: []corev1.VolumeMount{socketMount, refDataMount},
515550
},
551+
expectedHealthReporter("unix:///var/run/kmsplugin/kms-555.sock"),
516552
},
517553
Volumes: []corev1.Volume{f.resourceDirVolume, socketVolume, refDataVolume},
518554
},
@@ -540,7 +576,7 @@ func TestAddKMSPluginSidecarToPodSpec(t *testing.T) {
540576

541577
for _, tt := range tests {
542578
t.Run(tt.name, func(t *testing.T) {
543-
err := AddKMSPluginSidecarToPodSpec(context.Background(), tt.actualPodSpec, "kube-apiserver", "openshift-kube-apiserver", "encryption-config", tt.secretClient, tt.featureGateAccessor)
579+
err := AddKMSPluginSidecarToPodSpec(context.Background(), tt.actualPodSpec, "kube-apiserver", "openshift-kube-apiserver", "encryption-config", tt.secretClient, tt.featureGateAccessor, "cluster-kube-apiserver-operator", "quay.io/test/operator:latest")
544580
if tt.wantErr != "" {
545581
require.ErrorContains(t, err, tt.wantErr)
546582
return
@@ -565,7 +601,7 @@ func TestAddKMSPluginSidecarToPodSpecIdempotency(t *testing.T) {
565601

566602
call := func() {
567603
t.Helper()
568-
err := AddKMSPluginSidecarToPodSpec(context.Background(), podSpec, "kube-apiserver", "openshift-kube-apiserver", "encryption-config", sc, fga)
604+
err := AddKMSPluginSidecarToPodSpec(context.Background(), podSpec, "kube-apiserver", "openshift-kube-apiserver", "encryption-config", sc, fga, "cluster-kube-apiserver-operator", "quay.io/test/operator:latest")
569605
require.NoError(t, err)
570606
}
571607

0 commit comments

Comments
 (0)