From 2ef34dd7c042110d96f8dd4a2f27be584a2a809b Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 3 Mar 2026 13:23:09 -0500 Subject: [PATCH 01/31] feat(api): add canonical multi-backend-set model for api server lb --- api/v1beta1/types.go | 37 ++++++ api/v1beta1/types_test.go | 60 +++++++++ api/v1beta1/zz_generated.conversion.go | 38 ++++++ api/v1beta1/zz_generated.deepcopy.go | 23 ++++ api/v1beta2/types.go | 37 ++++++ api/v1beta2/types_test.go | 60 +++++++++ api/v1beta2/zz_generated.deepcopy.go | 23 ++++ ...tructure.cluster.x-k8s.io_ociclusters.yaml | 118 ++++++++++++++++- ....cluster.x-k8s.io_ociclustertemplates.yaml | 120 +++++++++++++++++- ...e.cluster.x-k8s.io_ocimanagedclusters.yaml | 118 ++++++++++++++++- ...r.x-k8s.io_ocimanagedclustertemplates.yaml | 120 +++++++++++++++++- 11 files changed, 738 insertions(+), 16 deletions(-) create mode 100644 api/v1beta1/types_test.go create mode 100644 api/v1beta2/types_test.go diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index af1ac3ad0..52ecd1c38 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -989,7 +989,15 @@ type LoadBalancer struct { // NLBSpec specifies the NLB spec. type NLBSpec struct { + // BackendSets specifies the canonical list of API server backend sets. + // When set, this field takes precedence over the legacy `backendSetDetails` field. + // +optional + // +listType=map + // +listMapKey=name + BackendSets []NLBBackendSet `json:"backendSets,omitempty"` + // BackendSetDetails specifies the configuration of a network load balancer backend set. + // Deprecated: use `backendSets` instead. // +optional BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` @@ -1000,6 +1008,16 @@ type NLBSpec struct { ReservedIpIds []string `json:"reservedIpIds,omitempty"` } +// NLBBackendSet specifies the configuration for a named network load balancer backend set. +type NLBBackendSet struct { + // Name is the API server backend set identifier. + Name string `json:"name"` + + // BackendSetDetails specifies the backend set behavior. + // +optional + BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` +} + // BackendSetDetails specifies the configuration of a network load balancer backend set. type BackendSetDetails struct { // If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. @@ -1031,6 +1049,25 @@ type HealthChecker struct { UrlPath *string `json:"urlPath,omitempty"` } +// CanonicalBackendSets returns the canonical API server backend set list. +// Precedence: +// 1. If `backendSets` is configured, it is used as-is. +// 2. Otherwise, a single backend set named APIServerLBBackendSetName is synthesized from the legacy `backendSetDetails`. +func (in *NLBSpec) CanonicalBackendSets() []NLBBackendSet { + if len(in.BackendSets) > 0 { + return append([]NLBBackendSet(nil), in.BackendSets...) + } + if in.BackendSetDetails == (BackendSetDetails{}) { + return nil + } + return []NLBBackendSet{ + { + Name: APIServerLBBackendSetName, + BackendSetDetails: in.BackendSetDetails, + }, + } +} + // NetworkSpec specifies what the OCI networking resources should look like. type NetworkSpec struct { // SkipNetworkManagement defines if the networking spec(VCN related) specified by the user needs to be reconciled(actioned-upon) diff --git a/api/v1beta1/types_test.go b/api/v1beta1/types_test.go new file mode 100644 index 000000000..7d9fc4cf5 --- /dev/null +++ b/api/v1beta1/types_test.go @@ -0,0 +1,60 @@ +package v1beta1 + +import ( + "encoding/json" + "testing" +) + +func TestNLBSpecCanonicalBackendSets_Precedence(t *testing.T) { + t.Run("uses backendSets when present", func(t *testing.T) { + spec := NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "new-set"}, + }, + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 || got[0].Name != "new-set" { + t.Fatalf("expected canonical backendSets to be used, got %#v", got) + } + }) + + t.Run("falls back to legacy backendSetDetails", func(t *testing.T) { + spec := NLBSpec{ + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 { + t.Fatalf("expected one synthesized backend set, got %#v", got) + } + if got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected synthesized backend set name %q, got %q", APIServerLBBackendSetName, got[0].Name) + } + if got[0].BackendSetDetails.IsFailOpen == nil || *got[0].BackendSetDetails.IsFailOpen != true { + t.Fatalf("expected legacy backend set details to be preserved, got %#v", got[0].BackendSetDetails) + } + }) +} + +func TestNLBSpecLegacyDecode_ToCanonical(t *testing.T) { + raw := []byte(`{"backendSetDetails":{"isFailOpen":true}}`) + var spec NLBSpec + if err := json.Unmarshal(raw, &spec); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 || got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected legacy decode to map to one canonical backend set, got %#v", got) + } +} + +func boolPtr(v bool) *bool { + return &v +} diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 279055d0f..3d5c38de7 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -370,6 +370,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*NLBBackendSet)(nil), (*v1beta2.NLBBackendSet)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(a.(*NLBBackendSet), b.(*v1beta2.NLBBackendSet), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.NLBBackendSet)(nil), (*NLBBackendSet)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet(a.(*v1beta2.NLBBackendSet), b.(*NLBBackendSet), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*NLBSpec)(nil), (*v1beta2.NLBSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_NLBSpec_To_v1beta2_NLBSpec(a.(*NLBSpec), b.(*v1beta2.NLBSpec), scope) }); err != nil { @@ -1860,7 +1870,34 @@ func autoConvert_v1beta2_LoadBalancer_To_v1beta1_LoadBalancer(in *v1beta2.LoadBa return nil } +func autoConvert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(in *NLBBackendSet, out *v1beta2.NLBBackendSet, s conversion.Scope) error { + out.Name = in.Name + if err := Convert_v1beta1_BackendSetDetails_To_v1beta2_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet is an autogenerated conversion function. +func Convert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(in *NLBBackendSet, out *v1beta2.NLBBackendSet, s conversion.Scope) error { + return autoConvert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(in, out, s) +} + +func autoConvert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet(in *v1beta2.NLBBackendSet, out *NLBBackendSet, s conversion.Scope) error { + out.Name = in.Name + if err := Convert_v1beta2_BackendSetDetails_To_v1beta1_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet is an autogenerated conversion function. +func Convert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet(in *v1beta2.NLBBackendSet, out *NLBBackendSet, s conversion.Scope) error { + return autoConvert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet(in, out, s) +} + func autoConvert_v1beta1_NLBSpec_To_v1beta2_NLBSpec(in *NLBSpec, out *v1beta2.NLBSpec, s conversion.Scope) error { + out.BackendSets = *(*[]v1beta2.NLBBackendSet)(unsafe.Pointer(&in.BackendSets)) if err := Convert_v1beta1_BackendSetDetails_To_v1beta2_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { return err } @@ -1874,6 +1911,7 @@ func Convert_v1beta1_NLBSpec_To_v1beta2_NLBSpec(in *NLBSpec, out *v1beta2.NLBSpe } func autoConvert_v1beta2_NLBSpec_To_v1beta1_NLBSpec(in *v1beta2.NLBSpec, out *NLBSpec, s conversion.Scope) error { + out.BackendSets = *(*[]NLBBackendSet)(unsafe.Pointer(&in.BackendSets)) if err := Convert_v1beta2_BackendSetDetails_To_v1beta1_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { return err } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0ea122fc6..fde862da4 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1105,9 +1105,32 @@ func (in *LoadBalancer) DeepCopy() *LoadBalancer { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NLBBackendSet) DeepCopyInto(out *NLBBackendSet) { + *out = *in + in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NLBBackendSet. +func (in *NLBBackendSet) DeepCopy() *NLBBackendSet { + if in == nil { + return nil + } + out := new(NLBBackendSet) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NLBSpec) DeepCopyInto(out *NLBSpec) { *out = *in + if in.BackendSets != nil { + in, out := &in.BackendSets, &out.BackendSets + *out = make([]NLBBackendSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) if in.ReservedIpIds != nil { in, out := &in.ReservedIpIds, &out.ReservedIpIds diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index 532ff82e9..4b6c7de91 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -998,7 +998,15 @@ type LoadBalancer struct { // NLBSpec specifies the NLB spec. type NLBSpec struct { + // BackendSets specifies the canonical list of API server backend sets. + // When set, this field takes precedence over the legacy `backendSetDetails` field. + // +optional + // +listType=map + // +listMapKey=name + BackendSets []NLBBackendSet `json:"backendSets,omitempty"` + // BackendSetDetails specifies the configuration of a network load balancer backend set. + // Deprecated: use `backendSets` instead. // +optional BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` @@ -1009,6 +1017,16 @@ type NLBSpec struct { ReservedIpIds []string `json:"reservedIpIds,omitempty"` } +// NLBBackendSet specifies the configuration for a named network load balancer backend set. +type NLBBackendSet struct { + // Name is the API server backend set identifier. + Name string `json:"name"` + + // BackendSetDetails specifies the backend set behavior. + // +optional + BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` +} + // BackendSetDetails specifies the configuration of a network load balancer backend set. type BackendSetDetails struct { // If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. @@ -1041,6 +1059,25 @@ type HealthChecker struct { UrlPath *string `json:"urlPath,omitempty"` } +// CanonicalBackendSets returns the canonical API server backend set list. +// Precedence: +// 1. If `backendSets` is configured, it is used as-is. +// 2. Otherwise, a single backend set named APIServerLBBackendSetName is synthesized from the legacy `backendSetDetails`. +func (in *NLBSpec) CanonicalBackendSets() []NLBBackendSet { + if len(in.BackendSets) > 0 { + return append([]NLBBackendSet(nil), in.BackendSets...) + } + if in.BackendSetDetails == (BackendSetDetails{}) { + return nil + } + return []NLBBackendSet{ + { + Name: APIServerLBBackendSetName, + BackendSetDetails: in.BackendSetDetails, + }, + } +} + // NetworkSpec specifies what the OCI networking resources should look like. type NetworkSpec struct { // SkipNetworkManagement defines if the networking spec(VCN related) specified by the user needs to be reconciled(actioned-upon) diff --git a/api/v1beta2/types_test.go b/api/v1beta2/types_test.go new file mode 100644 index 000000000..39c3814a1 --- /dev/null +++ b/api/v1beta2/types_test.go @@ -0,0 +1,60 @@ +package v1beta2 + +import ( + "encoding/json" + "testing" +) + +func TestNLBSpecCanonicalBackendSets_Precedence(t *testing.T) { + t.Run("uses backendSets when present", func(t *testing.T) { + spec := NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "new-set"}, + }, + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 || got[0].Name != "new-set" { + t.Fatalf("expected canonical backendSets to be used, got %#v", got) + } + }) + + t.Run("falls back to legacy backendSetDetails", func(t *testing.T) { + spec := NLBSpec{ + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 { + t.Fatalf("expected one synthesized backend set, got %#v", got) + } + if got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected synthesized backend set name %q, got %q", APIServerLBBackendSetName, got[0].Name) + } + if got[0].BackendSetDetails.IsFailOpen == nil || *got[0].BackendSetDetails.IsFailOpen != true { + t.Fatalf("expected legacy backend set details to be preserved, got %#v", got[0].BackendSetDetails) + } + }) +} + +func TestNLBSpecLegacyDecode_ToCanonical(t *testing.T) { + raw := []byte(`{"backendSetDetails":{"isFailOpen":true}}`) + var spec NLBSpec + if err := json.Unmarshal(raw, &spec); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 || got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected legacy decode to map to one canonical backend set, got %#v", got) + } +} + +func boolPtr(v bool) *bool { + return &v +} diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 4e4150c08..fb760e3b9 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -1347,9 +1347,32 @@ func (in *NATGateway) DeepCopy() *NATGateway { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NLBBackendSet) DeepCopyInto(out *NLBBackendSet) { + *out = *in + in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NLBBackendSet. +func (in *NLBBackendSet) DeepCopy() *NLBBackendSet { + if in == nil { + return nil + } + out := new(NLBBackendSet) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NLBSpec) DeepCopyInto(out *NLBSpec) { *out = *in + if in.BackendSets != nil { + in, out := &in.BackendSets, &out.BackendSets + *out = make([]NLBBackendSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) if in.ReservedIpIds != nil { in, out := &in.ReservedIpIds, &out.ReservedIpIds diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml index 072611e28..be2cf8883 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml @@ -133,8 +133,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections will @@ -165,6 +166,60 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend set. + properties: + backendSetDetails: + description: BackendSetDetails specifies the backend + set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + name: + description: Name is the API server backend set + identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID @@ -1294,8 +1349,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections will @@ -1326,6 +1382,60 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend set. + properties: + backendSetDetails: + description: BackendSetDetails specifies the backend + set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + name: + description: Name is the API server backend set + identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml index 9be0185ea..ec0d29c19 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml @@ -146,8 +146,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections @@ -180,6 +181,61 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend + set. + properties: + backendSetDetails: + description: BackendSetDetails specifies + the backend set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + name: + description: Name is the API server backend + set identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID @@ -1252,8 +1308,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections @@ -1286,6 +1343,61 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend + set. + properties: + backendSetDetails: + description: BackendSetDetails specifies + the backend set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + name: + description: Name is the API server backend + set identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml index eb15a2175..adf7cd19d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml @@ -136,8 +136,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections will @@ -168,6 +169,60 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend set. + properties: + backendSetDetails: + description: BackendSetDetails specifies the backend + set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + name: + description: Name is the API server backend set + identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID @@ -1300,8 +1355,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections will @@ -1332,6 +1388,60 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend set. + properties: + backendSetDetails: + description: BackendSetDetails specifies the backend + set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + name: + description: Name is the API server backend set + identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml index 1ff3babbe..de46e9bae 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml @@ -151,8 +151,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections @@ -185,6 +186,61 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend + set. + properties: + backendSetDetails: + description: BackendSetDetails specifies + the backend set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + name: + description: Name is the API server backend + set identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID @@ -1262,8 +1318,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections @@ -1296,6 +1353,61 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend + set. + properties: + backendSetDetails: + description: BackendSetDetails specifies + the backend set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + name: + description: Name is the API server backend + set identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID From ee5885cc368dfd373a9dd5275e26f0ce806b2c31 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 3 Mar 2026 13:27:09 -0500 Subject: [PATCH 02/31] feat(api): validate and test multi-backend-set conversion invariants --- api/v1beta1/ocicluster_conversion_test.go | 77 ++++++++++++++++++ api/v1beta1/types_test.go | 20 +++++ api/v1beta2/types_test.go | 20 +++++ api/v1beta2/validator.go | 44 ++++++++++ api/v1beta2/validator_backendsets_test.go | 98 +++++++++++++++++++++++ 5 files changed, 259 insertions(+) create mode 100644 api/v1beta2/validator_backendsets_test.go diff --git a/api/v1beta1/ocicluster_conversion_test.go b/api/v1beta1/ocicluster_conversion_test.go index 787ccd8a3..005d861f7 100644 --- a/api/v1beta1/ocicluster_conversion_test.go +++ b/api/v1beta1/ocicluster_conversion_test.go @@ -25,6 +25,83 @@ import ( "sigs.k8s.io/controller-runtime/pkg/conversion" ) +func TestOCICluster_BackendSetsRoundTrip(t *testing.T) { + t.Run("v1beta1 legacy -> v1beta2 -> v1beta1 keeps legacy shape", func(t *testing.T) { + src := &OCICluster{ + Spec: OCIClusterSpec{ + NetworkSpec: NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + }, + }, + } + + hub := &v1beta2.OCICluster{} + if err := src.ConvertTo(hub); err != nil { + t.Fatalf("convert to hub failed: %v", err) + } + if len(hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) != 0 { + t.Fatalf("expected no canonical backendSets to be synthesized during conversion, got %#v", hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) + } + if hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen == nil || !*hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen { + t.Fatalf("expected legacy backendSetDetails to be preserved on hub") + } + + restored := &OCICluster{} + if err := restored.ConvertFrom(hub); err != nil { + t.Fatalf("convert from hub failed: %v", err) + } + if restored.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen == nil || !*restored.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen { + t.Fatalf("expected legacy backendSetDetails after roundtrip, got %#v", restored.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails) + } + }) + + t.Run("v1beta2 canonical -> v1beta1 -> v1beta2 keeps canonical shape", func(t *testing.T) { + srcHub := &v1beta2.OCICluster{ + Spec: v1beta2.OCIClusterSpec{ + NetworkSpec: v1beta2.NetworkSpec{ + APIServerLB: v1beta2.LoadBalancer{ + NLBSpec: v1beta2.NLBSpec{ + BackendSets: []v1beta2.NLBBackendSet{ + { + Name: "new-set", + BackendSetDetails: v1beta2.BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + }, + }, + }, + }, + } + + spoke := &OCICluster{} + if err := spoke.ConvertFrom(srcHub); err != nil { + t.Fatalf("convert from hub failed: %v", err) + } + if len(spoke.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) != 1 || spoke.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].Name != "new-set" { + t.Fatalf("expected canonical backendSets after down-conversion, got %#v", spoke.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) + } + + restoredHub := &v1beta2.OCICluster{} + if err := spoke.ConvertTo(restoredHub); err != nil { + t.Fatalf("convert back to hub failed: %v", err) + } + if len(restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) != 1 || restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].Name != "new-set" { + t.Fatalf("expected canonical backendSets after hub roundtrip, got %#v", restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) + } + if restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].BackendSetDetails.IsFailOpen == nil || !*restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].BackendSetDetails.IsFailOpen { + t.Fatalf("expected backendSetDetails to remain intact after roundtrip") + } + }) +} + func TestOCICluster_ConvertTo(t *testing.T) { type fields struct { TypeMeta metav1.TypeMeta diff --git a/api/v1beta1/types_test.go b/api/v1beta1/types_test.go index 7d9fc4cf5..d6b343d4c 100644 --- a/api/v1beta1/types_test.go +++ b/api/v1beta1/types_test.go @@ -55,6 +55,26 @@ func TestNLBSpecLegacyDecode_ToCanonical(t *testing.T) { } } +func TestNLBSpecDeepCopy_NoSharedPointers(t *testing.T) { + spec := &NLBSpec{ + BackendSets: []NLBBackendSet{ + { + Name: "set-a", + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + } + + cp := spec.DeepCopy() + *cp.BackendSets[0].BackendSetDetails.IsFailOpen = false + + if spec.BackendSets[0].BackendSetDetails.IsFailOpen == nil || *spec.BackendSets[0].BackendSetDetails.IsFailOpen != true { + t.Fatalf("expected deepcopy to isolate nested pointers, original=%#v", spec.BackendSets[0].BackendSetDetails) + } +} + func boolPtr(v bool) *bool { return &v } diff --git a/api/v1beta2/types_test.go b/api/v1beta2/types_test.go index 39c3814a1..62d0a44ae 100644 --- a/api/v1beta2/types_test.go +++ b/api/v1beta2/types_test.go @@ -55,6 +55,26 @@ func TestNLBSpecLegacyDecode_ToCanonical(t *testing.T) { } } +func TestNLBSpecDeepCopy_NoSharedPointers(t *testing.T) { + spec := &NLBSpec{ + BackendSets: []NLBBackendSet{ + { + Name: "set-a", + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + } + + cp := spec.DeepCopy() + *cp.BackendSets[0].BackendSetDetails.IsFailOpen = false + + if spec.BackendSets[0].BackendSetDetails.IsFailOpen == nil || *spec.BackendSets[0].BackendSetDetails.IsFailOpen != true { + t.Fatalf("expected deepcopy to isolate nested pointers, original=%#v", spec.BackendSets[0].BackendSetDetails) + } +} + func boolPtr(v bool) *bool { return &v } diff --git a/api/v1beta2/validator.go b/api/v1beta2/validator.go index 97854df10..30a363c3b 100644 --- a/api/v1beta2/validator.go +++ b/api/v1beta2/validator.go @@ -23,6 +23,7 @@ import ( "fmt" "net" "regexp" + "strings" "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" "k8s.io/apimachinery/pkg/util/validation/field" @@ -51,6 +52,7 @@ const ( sourceRequiredFormat = "invalid %s: Source may not be empty" protocolRequiredFormat = "invalid %s: Protocol may not be empty" invalidCIDRFormatFormat = "invalid %s: CIDR format" + apiServerBackendSetNameRegex = `^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$` ) // invalidNameRegex is a broad regex used to validate allows names in OCI @@ -126,6 +128,7 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network nsgErrors := validateNSGs(validRoles, networkSpec.Vcn.NetworkSecurityGroup.List, fldPath.Child("networkSecurityGroups")) allErrs = append(allErrs, nsgErrors...) } + allErrs = append(allErrs, validateAPIServerLBBackendSets(networkSpec.APIServerLB.NLBSpec, fldPath.Child("apiServerLoadBalancer").Child("nlbSpec"))...) if len(allErrs) == 0 { return nil @@ -133,6 +136,47 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network return allErrs } +func validateAPIServerLBBackendSets(spec NLBSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + legacyConfigured := spec.BackendSetDetails != (BackendSetDetails{}) + if len(spec.BackendSets) > 0 && legacyConfigured { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child("backendSetDetails"), + "cannot be set when backendSets is configured; move legacy backendSetDetails into backendSets", + )) + } + + nameRegex := regexp.MustCompile(apiServerBackendSetNameRegex) + seen := map[string]int{} + for i, backendSet := range spec.BackendSets { + namePath := fldPath.Child("backendSets").Index(i).Child("name") + name := strings.TrimSpace(backendSet.Name) + if name == "" { + allErrs = append(allErrs, field.Required(namePath, "must not be empty")) + continue + } + if !nameRegex.MatchString(name) { + allErrs = append(allErrs, field.Invalid( + namePath, + backendSet.Name, + "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$ (1-32 chars; alphanumeric, '-', '_')", + )) + } + if firstIdx, ok := seen[name]; ok { + allErrs = append(allErrs, field.Invalid( + namePath, + backendSet.Name, + fmt.Sprintf("duplicate backend set name, already used at backendSets[%d].name", firstIdx), + )) + continue + } + seen[name] = i + } + + return allErrs +} + // validateVCNCIDR validates the CIDR of a VNC. func validateVCNCIDR(vncCIDR string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta2/validator_backendsets_test.go b/api/v1beta2/validator_backendsets_test.go new file mode 100644 index 000000000..8c4488bb3 --- /dev/null +++ b/api/v1beta2/validator_backendsets_test.go @@ -0,0 +1,98 @@ +package v1beta2 + +import ( + "strings" + "testing" + + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func TestValidateNetworkSpec_BackendSets(t *testing.T) { + t.Run("rejects mixed legacy and canonical fields", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "new-set"}}, + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + }, + NetworkSpec{}, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for mixed fields") + } + if !strings.Contains(errs[0].Error(), "backendSetDetails") { + t.Fatalf("expected backendSetDetails path in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects duplicate backend set names", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "dup"}, {Name: "dup"}}, + }, + }, + }, + NetworkSpec{}, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate names") + } + if !strings.Contains(errs[0].Error(), "duplicate backend set name") { + t.Fatalf("expected duplicate-name guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects invalid backend set name format", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "bad name"}}, + }, + }, + }, + NetworkSpec{}, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for invalid name format") + } + if !strings.Contains(errs[0].Error(), "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$") { + t.Fatalf("expected regex guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts canonical backend sets", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "apiserver_a"}, {Name: "apiserver-b"}}, + }, + }, + }, + NetworkSpec{}, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) +} From 0cb279f8a49d80c9ff88e1df29d5246a62d7e145 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 3 Mar 2026 13:35:09 -0500 Subject: [PATCH 03/31] feat(scope): build multi-backend-set create configs for api server lb --- api/v1beta1/types.go | 3 - api/v1beta1/types_test.go | 12 +++ api/v1beta2/types.go | 3 - api/v1beta2/types_test.go | 12 +++ cloud/scope/apiserver_lb_backendsets_test.go | 59 +++++++++++++++ cloud/scope/load_balancer_reconciler.go | 45 +++++++---- .../scope/network_load_balancer_reconciler.go | 74 +++++++++++-------- 7 files changed, 157 insertions(+), 51 deletions(-) create mode 100644 cloud/scope/apiserver_lb_backendsets_test.go diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 52ecd1c38..02b22b3ba 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -1057,9 +1057,6 @@ func (in *NLBSpec) CanonicalBackendSets() []NLBBackendSet { if len(in.BackendSets) > 0 { return append([]NLBBackendSet(nil), in.BackendSets...) } - if in.BackendSetDetails == (BackendSetDetails{}) { - return nil - } return []NLBBackendSet{ { Name: APIServerLBBackendSetName, diff --git a/api/v1beta1/types_test.go b/api/v1beta1/types_test.go index d6b343d4c..39dd18733 100644 --- a/api/v1beta1/types_test.go +++ b/api/v1beta1/types_test.go @@ -40,6 +40,18 @@ func TestNLBSpecCanonicalBackendSets_Precedence(t *testing.T) { t.Fatalf("expected legacy backend set details to be preserved, got %#v", got[0].BackendSetDetails) } }) + + t.Run("defaults to legacy backend set name when empty", func(t *testing.T) { + spec := NLBSpec{} + + got := spec.CanonicalBackendSets() + if len(got) != 1 { + t.Fatalf("expected one synthesized backend set, got %#v", got) + } + if got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected synthesized backend set name %q, got %q", APIServerLBBackendSetName, got[0].Name) + } + }) } func TestNLBSpecLegacyDecode_ToCanonical(t *testing.T) { diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index 4b6c7de91..1c10cb27c 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -1067,9 +1067,6 @@ func (in *NLBSpec) CanonicalBackendSets() []NLBBackendSet { if len(in.BackendSets) > 0 { return append([]NLBBackendSet(nil), in.BackendSets...) } - if in.BackendSetDetails == (BackendSetDetails{}) { - return nil - } return []NLBBackendSet{ { Name: APIServerLBBackendSetName, diff --git a/api/v1beta2/types_test.go b/api/v1beta2/types_test.go index 62d0a44ae..4c73e692a 100644 --- a/api/v1beta2/types_test.go +++ b/api/v1beta2/types_test.go @@ -40,6 +40,18 @@ func TestNLBSpecCanonicalBackendSets_Precedence(t *testing.T) { t.Fatalf("expected legacy backend set details to be preserved, got %#v", got[0].BackendSetDetails) } }) + + t.Run("defaults to legacy backend set name when empty", func(t *testing.T) { + spec := NLBSpec{} + + got := spec.CanonicalBackendSets() + if len(got) != 1 { + t.Fatalf("expected one synthesized backend set, got %#v", got) + } + if got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected synthesized backend set name %q, got %q", APIServerLBBackendSetName, got[0].Name) + } + }) } func TestNLBSpecLegacyDecode_ToCanonical(t *testing.T) { diff --git a/cloud/scope/apiserver_lb_backendsets_test.go b/cloud/scope/apiserver_lb_backendsets_test.go new file mode 100644 index 000000000..f5f8cde2a --- /dev/null +++ b/cloud/scope/apiserver_lb_backendsets_test.go @@ -0,0 +1,59 @@ +package scope + +import ( + "testing" + + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" + "github.com/oracle/oci-go-sdk/v65/common" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" +) + +func TestBuildDesiredNLBListenersAndBackendSets(t *testing.T) { + scope := &ClusterScope{Cluster: &clusterv1.Cluster{}} + lb := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: "primary-set"}, + { + Name: "rollout-set", + BackendSetDetails: infrastructurev1beta2.BackendSetDetails{ + HealthChecker: infrastructurev1beta2.HealthChecker{ + UrlPath: common.String("/readyz"), + }, + }, + }, + }, + }, + } + + listeners, backendSets := scope.buildDesiredNLBListenersAndBackendSets(lb) + if len(backendSets) != 2 { + t.Fatalf("expected two backend sets, got %#v", backendSets) + } + if listeners[APIServerLBListener].DefaultBackendSetName == nil || *listeners[APIServerLBListener].DefaultBackendSetName != "primary-set" { + t.Fatalf("expected listener to reference first backend set, got %#v", listeners[APIServerLBListener]) + } + if backendSets["rollout-set"].HealthChecker == nil || backendSets["rollout-set"].HealthChecker.UrlPath == nil || *backendSets["rollout-set"].HealthChecker.UrlPath != "/readyz" { + t.Fatalf("expected rollout backend set health checker override, got %#v", backendSets["rollout-set"]) + } +} + +func TestBuildDesiredLBListenersAndBackendSets(t *testing.T) { + scope := &ClusterScope{Cluster: &clusterv1.Cluster{}} + lb := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: "primary-set"}, + {Name: "rollout-set"}, + }, + }, + } + + listeners, backendSets := scope.buildDesiredLBListenersAndBackendSets(lb) + if len(backendSets) != 2 { + t.Fatalf("expected two backend sets, got %#v", backendSets) + } + if listeners[APIServerLBListener].DefaultBackendSetName == nil || *listeners[APIServerLBListener].DefaultBackendSetName != "primary-set" { + t.Fatalf("expected listener to reference first backend set, got %#v", listeners[APIServerLBListener]) + } +} diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index e7b51fe3b..b4e8c4a33 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -145,21 +145,7 @@ func (s *ClusterScope) UpdateLB(ctx context.Context, lb infrastructurev1beta2.Lo // See https://docs.oracle.com/en-us/iaas/Content/LoadBalancer/overview.htm for more details on the Network // Load Balancer func (s *ClusterScope) CreateLB(ctx context.Context, lb infrastructurev1beta2.LoadBalancer) (*string, *string, error) { - listenerDetails := make(map[string]loadbalancer.ListenerDetails) - listenerDetails[APIServerLBListener] = loadbalancer.ListenerDetails{ - Protocol: common.String("TCP"), - Port: common.Int(int(s.APIServerPort())), - DefaultBackendSetName: common.String(APIServerLBBackendSetName), - } - backendSetDetails := make(map[string]loadbalancer.BackendSetDetails) - backendSetDetails[APIServerLBBackendSetName] = loadbalancer.BackendSetDetails{ - Policy: common.String("ROUND_ROBIN"), - HealthChecker: &loadbalancer.HealthCheckerDetails{ - Port: common.Int(int(s.APIServerPort())), - Protocol: common.String("TCP"), - }, - Backends: []loadbalancer.BackendDetails{}, - } + listenerDetails, backendSetDetails := s.buildDesiredLBListenersAndBackendSets(lb) var controlPlaneEndpointSubnets []string for _, subnet := range ptr.ToSubnetSlice(s.OCIClusterAccessor.GetNetworkSpec().Vcn.Subnets) { if subnet.ID != nil && subnet.Role == infrastructurev1beta2.ControlPlaneEndpointRole { @@ -225,6 +211,35 @@ func (s *ClusterScope) CreateLB(ctx context.Context, lb infrastructurev1beta2.Lo return lbs.Id, lbIp, nil } +func (s *ClusterScope) buildDesiredLBListenersAndBackendSets(lb infrastructurev1beta2.LoadBalancer) (map[string]loadbalancer.ListenerDetails, map[string]loadbalancer.BackendSetDetails) { + canonicalBackendSets := lb.NLBSpec.CanonicalBackendSets() + if len(canonicalBackendSets) == 0 { + canonicalBackendSets = []infrastructurev1beta2.NLBBackendSet{{Name: APIServerLBBackendSetName}} + } + + backendSetDetails := make(map[string]loadbalancer.BackendSetDetails, len(canonicalBackendSets)) + for _, backendSet := range canonicalBackendSets { + backendSetDetails[backendSet.Name] = loadbalancer.BackendSetDetails{ + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthCheckerDetails{ + Port: common.Int(int(s.APIServerPort())), + Protocol: common.String("TCP"), + }, + Backends: []loadbalancer.BackendDetails{}, + } + } + + primaryBackendSetName := canonicalBackendSets[0].Name + listenerDetails := map[string]loadbalancer.ListenerDetails{ + APIServerLBListener: { + Protocol: common.String("TCP"), + Port: common.Int(int(s.APIServerPort())), + DefaultBackendSetName: common.String(primaryBackendSetName), + }, + } + return listenerDetails, backendSetDetails +} + func (s *ClusterScope) getLoadbalancerIp(lb loadbalancer.LoadBalancer) (*string, error) { var lbIp *string if len(lb.IpAddresses) < 1 { diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index 9736a5122..04a571235 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -144,36 +144,7 @@ func (s *ClusterScope) UpdateNLB(ctx context.Context, nlb infrastructurev1beta2. // See https://docs.oracle.com/en-us/iaas/Content/NetworkLoadBalancer/overview.htm for more details on the Network // Load Balancer func (s *ClusterScope) CreateNLB(ctx context.Context, lb infrastructurev1beta2.LoadBalancer) (*string, *string, error) { - isPreserverSourceIp := lb.NLBSpec.BackendSetDetails.IsPreserveSource - if isPreserverSourceIp == nil { - isPreserverSourceIp = common.Bool(false) - } - listenerDetails := make(map[string]networkloadbalancer.ListenerDetails) - listenerDetails[APIServerLBListener] = networkloadbalancer.ListenerDetails{ - Protocol: networkloadbalancer.ListenerProtocolsTcp, - Port: common.Int(int(s.APIServerPort())), - DefaultBackendSetName: common.String(APIServerLBBackendSetName), - Name: common.String(APIServerLBListener), - } - - backendSetDetails := make(map[string]networkloadbalancer.BackendSetDetails) - healthCheckUrl := lb.NLBSpec.BackendSetDetails.HealthChecker.UrlPath - if healthCheckUrl == nil { - healthCheckUrl = common.String("/healthz") - } - backendSetDetails[APIServerLBBackendSetName] = networkloadbalancer.BackendSetDetails{ - Policy: LoadBalancerPolicy, - IsPreserveSource: isPreserverSourceIp, - IsFailOpen: lb.NLBSpec.BackendSetDetails.IsFailOpen, - IsInstantFailoverEnabled: lb.NLBSpec.BackendSetDetails.IsInstantFailoverEnabled, - HealthChecker: &networkloadbalancer.HealthChecker{ - Port: common.Int(int(s.APIServerPort())), - Protocol: networkloadbalancer.HealthCheckProtocolsHttps, - UrlPath: healthCheckUrl, - ReturnCode: common.Int(200), - }, - Backends: []networkloadbalancer.Backend{}, - } + listenerDetails, backendSetDetails := s.buildDesiredNLBListenersAndBackendSets(lb) var controlPlaneEndpointSubnets []string for _, subnet := range ptr.ToSubnetSlice(s.OCIClusterAccessor.GetNetworkSpec().Vcn.Subnets) { @@ -248,6 +219,49 @@ func (s *ClusterScope) CreateNLB(ctx context.Context, lb infrastructurev1beta2.L return nlb.Id, nlbIp, nil } +func (s *ClusterScope) buildDesiredNLBListenersAndBackendSets(lb infrastructurev1beta2.LoadBalancer) (map[string]networkloadbalancer.ListenerDetails, map[string]networkloadbalancer.BackendSetDetails) { + canonicalBackendSets := lb.NLBSpec.CanonicalBackendSets() + if len(canonicalBackendSets) == 0 { + canonicalBackendSets = []infrastructurev1beta2.NLBBackendSet{{Name: APIServerLBBackendSetName}} + } + + backendSetDetails := make(map[string]networkloadbalancer.BackendSetDetails, len(canonicalBackendSets)) + for _, backendSet := range canonicalBackendSets { + isPreserveSourceIp := backendSet.BackendSetDetails.IsPreserveSource + if isPreserveSourceIp == nil { + isPreserveSourceIp = common.Bool(false) + } + healthCheckURL := backendSet.BackendSetDetails.HealthChecker.UrlPath + if healthCheckURL == nil { + healthCheckURL = common.String("/healthz") + } + backendSetDetails[backendSet.Name] = networkloadbalancer.BackendSetDetails{ + Policy: LoadBalancerPolicy, + IsPreserveSource: isPreserveSourceIp, + IsFailOpen: backendSet.BackendSetDetails.IsFailOpen, + IsInstantFailoverEnabled: backendSet.BackendSetDetails.IsInstantFailoverEnabled, + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(int(s.APIServerPort())), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: healthCheckURL, + ReturnCode: common.Int(200), + }, + Backends: []networkloadbalancer.Backend{}, + } + } + + primaryBackendSetName := canonicalBackendSets[0].Name + listenerDetails := map[string]networkloadbalancer.ListenerDetails{ + APIServerLBListener: { + Protocol: networkloadbalancer.ListenerProtocolsTcp, + Port: common.Int(int(s.APIServerPort())), + DefaultBackendSetName: common.String(primaryBackendSetName), + Name: common.String(APIServerLBListener), + }, + } + return listenerDetails, backendSetDetails +} + func (s *ClusterScope) getNetworkLoadbalancerIp(nlb networkloadbalancer.NetworkLoadBalancer) (*string, error) { var nlbIp *string if len(nlb.IpAddresses) < 1 { From beeda3d14d8a84cc2a8512616e7fa7bf5d5d8dfb Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 3 Mar 2026 14:30:43 -0500 Subject: [PATCH 04/31] feat(scope): add multi-listener mapping for api server backend sets --- api/v1beta1/types.go | 5 +++ api/v1beta1/zz_generated.conversion.go | 2 ++ api/v1beta1/zz_generated.deepcopy.go | 5 +++ api/v1beta2/types.go | 5 +++ api/v1beta2/validator.go | 22 +++++++++++++ api/v1beta2/validator_backendsets_test.go | 29 ++++++++++++++++ api/v1beta2/zz_generated.deepcopy.go | 5 +++ cloud/scope/apiserver_lb_backendsets_test.go | 17 +++++++++- cloud/scope/apiserver_listener_naming.go | 33 +++++++++++++++++++ cloud/scope/load_balancer_reconciler.go | 14 ++++---- .../scope/network_load_balancer_reconciler.go | 16 +++++---- ...tructure.cluster.x-k8s.io_ociclusters.yaml | 12 +++++++ ....cluster.x-k8s.io_ociclustertemplates.yaml | 12 +++++++ ...e.cluster.x-k8s.io_ocimanagedclusters.yaml | 12 +++++++ ...r.x-k8s.io_ocimanagedclustertemplates.yaml | 12 +++++++ 15 files changed, 187 insertions(+), 14 deletions(-) create mode 100644 cloud/scope/apiserver_listener_naming.go diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 02b22b3ba..ad0eacd0a 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -1013,6 +1013,11 @@ type NLBBackendSet struct { // Name is the API server backend set identifier. Name string `json:"name"` + // ListenerPort is the load balancer listener port routed to this backend set. + // When omitted, the cluster API server port is used. + // +optional + ListenerPort *int32 `json:"listenerPort,omitempty"` + // BackendSetDetails specifies the backend set behavior. // +optional BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 3d5c38de7..9770a853f 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -1872,6 +1872,7 @@ func autoConvert_v1beta2_LoadBalancer_To_v1beta1_LoadBalancer(in *v1beta2.LoadBa func autoConvert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(in *NLBBackendSet, out *v1beta2.NLBBackendSet, s conversion.Scope) error { out.Name = in.Name + out.ListenerPort = (*int32)(unsafe.Pointer(in.ListenerPort)) if err := Convert_v1beta1_BackendSetDetails_To_v1beta2_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { return err } @@ -1885,6 +1886,7 @@ func Convert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(in *NLBBackendSet, o func autoConvert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet(in *v1beta2.NLBBackendSet, out *NLBBackendSet, s conversion.Scope) error { out.Name = in.Name + out.ListenerPort = (*int32)(unsafe.Pointer(in.ListenerPort)) if err := Convert_v1beta2_BackendSetDetails_To_v1beta1_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { return err } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index fde862da4..60aef0fc5 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1108,6 +1108,11 @@ func (in *LoadBalancer) DeepCopy() *LoadBalancer { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NLBBackendSet) DeepCopyInto(out *NLBBackendSet) { *out = *in + if in.ListenerPort != nil { + in, out := &in.ListenerPort, &out.ListenerPort + *out = new(int32) + **out = **in + } in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) } diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index 1c10cb27c..9d9799c18 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -1022,6 +1022,11 @@ type NLBBackendSet struct { // Name is the API server backend set identifier. Name string `json:"name"` + // ListenerPort is the load balancer listener port routed to this backend set. + // When omitted, the cluster API server port is used. + // +optional + ListenerPort *int32 `json:"listenerPort,omitempty"` + // BackendSetDetails specifies the backend set behavior. // +optional BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` diff --git a/api/v1beta2/validator.go b/api/v1beta2/validator.go index 30a363c3b..9f67cc442 100644 --- a/api/v1beta2/validator.go +++ b/api/v1beta2/validator.go @@ -174,6 +174,28 @@ func validateAPIServerLBBackendSets(spec NLBSpec, fldPath *field.Path) field.Err seen[name] = i } + usedPorts := map[int32]int{} + for i, backendSet := range spec.BackendSets { + portPath := fldPath.Child("backendSets").Index(i).Child("listenerPort") + if backendSet.ListenerPort == nil { + continue + } + port := *backendSet.ListenerPort + if port < 1 || port > 65535 { + allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) + continue + } + if firstIdx, ok := usedPorts[port]; ok { + allErrs = append(allErrs, field.Invalid( + portPath, + port, + fmt.Sprintf("duplicate listenerPort, already used at backendSets[%d].listenerPort", firstIdx), + )) + continue + } + usedPorts[port] = i + } + return allErrs } diff --git a/api/v1beta2/validator_backendsets_test.go b/api/v1beta2/validator_backendsets_test.go index 8c4488bb3..d7b15115e 100644 --- a/api/v1beta2/validator_backendsets_test.go +++ b/api/v1beta2/validator_backendsets_test.go @@ -95,4 +95,33 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) } }) + + t.Run("rejects duplicate listener ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a", ListenerPort: int32Ptr(9345)}, + {Name: "set-b", ListenerPort: int32Ptr(9345)}, + }, + }, + }, + }, + NetworkSpec{}, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate listener ports") + } + if !strings.Contains(errs[0].Error(), "duplicate listenerPort") { + t.Fatalf("expected duplicate listener port guidance in error, got %q", errs[0].Error()) + } + }) +} + +func int32Ptr(v int32) *int32 { + return &v } diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index fb760e3b9..b3084eddf 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -1350,6 +1350,11 @@ func (in *NATGateway) DeepCopy() *NATGateway { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NLBBackendSet) DeepCopyInto(out *NLBBackendSet) { *out = *in + if in.ListenerPort != nil { + in, out := &in.ListenerPort, &out.ListenerPort + *out = new(int32) + **out = **in + } in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) } diff --git a/cloud/scope/apiserver_lb_backendsets_test.go b/cloud/scope/apiserver_lb_backendsets_test.go index f5f8cde2a..5d6115262 100644 --- a/cloud/scope/apiserver_lb_backendsets_test.go +++ b/cloud/scope/apiserver_lb_backendsets_test.go @@ -15,7 +15,8 @@ func TestBuildDesiredNLBListenersAndBackendSets(t *testing.T) { BackendSets: []infrastructurev1beta2.NLBBackendSet{ {Name: "primary-set"}, { - Name: "rollout-set", + Name: "rollout-set", + ListenerPort: int32Ptr(9345), BackendSetDetails: infrastructurev1beta2.BackendSetDetails{ HealthChecker: infrastructurev1beta2.HealthChecker{ UrlPath: common.String("/readyz"), @@ -30,14 +31,25 @@ func TestBuildDesiredNLBListenersAndBackendSets(t *testing.T) { if len(backendSets) != 2 { t.Fatalf("expected two backend sets, got %#v", backendSets) } + if len(listeners) != 2 { + t.Fatalf("expected two listeners, got %#v", listeners) + } if listeners[APIServerLBListener].DefaultBackendSetName == nil || *listeners[APIServerLBListener].DefaultBackendSetName != "primary-set" { t.Fatalf("expected listener to reference first backend set, got %#v", listeners[APIServerLBListener]) } + secondaryListenerName := desiredAPIServerListenerName(1, 2, "rollout-set") + if listeners[secondaryListenerName].Port == nil || *listeners[secondaryListenerName].Port != 9345 { + t.Fatalf("expected secondary listener to use custom port, got %#v", listeners[secondaryListenerName]) + } if backendSets["rollout-set"].HealthChecker == nil || backendSets["rollout-set"].HealthChecker.UrlPath == nil || *backendSets["rollout-set"].HealthChecker.UrlPath != "/readyz" { t.Fatalf("expected rollout backend set health checker override, got %#v", backendSets["rollout-set"]) } } +func int32Ptr(v int32) *int32 { + return &v +} + func TestBuildDesiredLBListenersAndBackendSets(t *testing.T) { scope := &ClusterScope{Cluster: &clusterv1.Cluster{}} lb := infrastructurev1beta2.LoadBalancer{ @@ -53,6 +65,9 @@ func TestBuildDesiredLBListenersAndBackendSets(t *testing.T) { if len(backendSets) != 2 { t.Fatalf("expected two backend sets, got %#v", backendSets) } + if len(listeners) != 2 { + t.Fatalf("expected two listeners, got %#v", listeners) + } if listeners[APIServerLBListener].DefaultBackendSetName == nil || *listeners[APIServerLBListener].DefaultBackendSetName != "primary-set" { t.Fatalf("expected listener to reference first backend set, got %#v", listeners[APIServerLBListener]) } diff --git a/cloud/scope/apiserver_listener_naming.go b/cloud/scope/apiserver_listener_naming.go new file mode 100644 index 000000000..b075cc953 --- /dev/null +++ b/cloud/scope/apiserver_listener_naming.go @@ -0,0 +1,33 @@ +package scope + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" +) + +func desiredAPIServerListenerPort(defaultPort int32, backendSet infrastructurev1beta2.NLBBackendSet) int32 { + if backendSet.ListenerPort != nil { + return *backendSet.ListenerPort + } + return defaultPort +} + +func desiredAPIServerListenerName(index int, total int, backendSetName string) string { + // Preserve legacy name for legacy/single-backend-set behavior. + if total == 1 && backendSetName == APIServerLBBackendSetName { + return APIServerLBListener + } + // Keep the first listener stable for the control plane endpoint. + if index == 0 { + return APIServerLBListener + } + return fmt.Sprintf("%s-%s", APIServerLBListener, stableShortHash(backendSetName)) +} + +func stableShortHash(value string) string { + sum := sha1.Sum([]byte(value)) + return hex.EncodeToString(sum[:])[:8] +} diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index b4e8c4a33..b6babc3af 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -229,13 +229,15 @@ func (s *ClusterScope) buildDesiredLBListenersAndBackendSets(lb infrastructurev1 } } - primaryBackendSetName := canonicalBackendSets[0].Name - listenerDetails := map[string]loadbalancer.ListenerDetails{ - APIServerLBListener: { + listenerDetails := make(map[string]loadbalancer.ListenerDetails, len(canonicalBackendSets)) + for i, backendSet := range canonicalBackendSets { + listenerName := desiredAPIServerListenerName(i, len(canonicalBackendSets), backendSet.Name) + port := desiredAPIServerListenerPort(s.APIServerPort(), backendSet) + listenerDetails[listenerName] = loadbalancer.ListenerDetails{ Protocol: common.String("TCP"), - Port: common.Int(int(s.APIServerPort())), - DefaultBackendSetName: common.String(primaryBackendSetName), - }, + Port: common.Int(int(port)), + DefaultBackendSetName: common.String(backendSet.Name), + } } return listenerDetails, backendSetDetails } diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index 04a571235..04e600fea 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -250,14 +250,16 @@ func (s *ClusterScope) buildDesiredNLBListenersAndBackendSets(lb infrastructurev } } - primaryBackendSetName := canonicalBackendSets[0].Name - listenerDetails := map[string]networkloadbalancer.ListenerDetails{ - APIServerLBListener: { + listenerDetails := make(map[string]networkloadbalancer.ListenerDetails, len(canonicalBackendSets)) + for i, backendSet := range canonicalBackendSets { + listenerName := desiredAPIServerListenerName(i, len(canonicalBackendSets), backendSet.Name) + port := desiredAPIServerListenerPort(s.APIServerPort(), backendSet) + listenerDetails[listenerName] = networkloadbalancer.ListenerDetails{ Protocol: networkloadbalancer.ListenerProtocolsTcp, - Port: common.Int(int(s.APIServerPort())), - DefaultBackendSetName: common.String(primaryBackendSetName), - Name: common.String(APIServerLBListener), - }, + Port: common.Int(int(port)), + DefaultBackendSetName: common.String(backendSet.Name), + Name: common.String(listenerName), + } } return listenerDetails, backendSetDetails } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml index be2cf8883..11e3b24c6 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml @@ -209,6 +209,12 @@ spec: The value is false by default. type: boolean type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer name: description: Name is the API server backend set identifier. @@ -1425,6 +1431,12 @@ spec: The value is false by default. type: boolean type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer name: description: Name is the API server backend set identifier. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml index ec0d29c19..4d3d0c088 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml @@ -225,6 +225,12 @@ spec: The value is false by default. type: boolean type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer name: description: Name is the API server backend set identifier. @@ -1387,6 +1393,12 @@ spec: The value is false by default. type: boolean type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer name: description: Name is the API server backend set identifier. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml index adf7cd19d..6435364ad 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml @@ -212,6 +212,12 @@ spec: The value is false by default. type: boolean type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer name: description: Name is the API server backend set identifier. @@ -1431,6 +1437,12 @@ spec: The value is false by default. type: boolean type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer name: description: Name is the API server backend set identifier. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml index de46e9bae..737aec6d2 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml @@ -230,6 +230,12 @@ spec: The value is false by default. type: boolean type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer name: description: Name is the API server backend set identifier. @@ -1397,6 +1403,12 @@ spec: The value is false by default. type: boolean type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer name: description: Name is the API server backend set identifier. From 4a1f0b9022319d7af9360e14b908ee790f02cc8f Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 3 Mar 2026 14:38:46 -0500 Subject: [PATCH 05/31] feat(scope): reconcile machine backend membership across backend sets --- cloud/scope/machine.go | 233 ++++++++++++++---------- cloud/scope/machine_backendsets_test.go | 199 ++++++++++++++++++++ 2 files changed, 332 insertions(+), 100 deletions(-) create mode 100644 cloud/scope/machine_backendsets_test.go diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index 2048ad840..bed3a5ca8 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -634,35 +634,43 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { if err != nil { return err } - backendSet := lb.BackendSets[APIServerLBBackendSetName] // When creating a LB, there is no way to set the backend Name, default backend name is the instance IP and port // So we use default backend name instead of machine name backendName := instanceIp + ":" + strconv.Itoa(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)) - if !m.containsLBBackend(backendSet, backendName) { - logger := m.Logger.WithValues("backend-set", *backendSet.Name) - logger.Info("Checking work request status for create backend") - // we always try to create the backend if it exists during a reconcile loop and wait for the work request - // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the - // work request, the create backend call may throw an error which is ok, as reconcile will go into - // an exponential backoff - resp, err := m.LoadBalancerClient.CreateBackend(ctx, loadbalancer.CreateBackendRequest{ - LoadBalancerId: loadbalancerId, - BackendSetName: backendSet.Name, - CreateBackendDetails: loadbalancer.CreateBackendDetails{ - IpAddress: common.String(instanceIp), - Port: common.Int(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)), - }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", string(m.OCIMachine.UID)), - }) - if err != nil { - return err + for _, backendSetName := range m.desiredAPIServerBackendSetNames() { + backendSet, ok := lb.BackendSets[backendSetName] + if !ok { + return errors.Errorf("backend set %q not found on load balancer", backendSetName) } - m.OCIMachine.Status.CreateBackendWorkRequestId = *resp.OpcWorkRequestId - logger.Info("Add instance to LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) - logger.Info("Waiting for LB work request to be complete") - _, err = ociutil.AwaitLBWorkRequest(ctx, m.LoadBalancerClient, resp.OpcWorkRequestId) - if err != nil { - return err + if backendSet.Name == nil { + backendSet.Name = common.String(backendSetName) + } + if !m.containsLBBackend(backendSet, backendName) { + logger := m.Logger.WithValues("backend-set", *backendSet.Name) + logger.Info("Checking work request status for create backend") + // we always try to create the backend if it exists during a reconcile loop and wait for the work request + // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the + // work request, the create backend call may throw an error which is ok, as reconcile will go into + // an exponential backoff + resp, err := m.LoadBalancerClient.CreateBackend(ctx, loadbalancer.CreateBackendRequest{ + LoadBalancerId: loadbalancerId, + BackendSetName: backendSet.Name, + CreateBackendDetails: loadbalancer.CreateBackendDetails{ + IpAddress: common.String(instanceIp), + Port: common.Int(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", string(m.OCIMachine.UID)), + }) + if err != nil { + return err + } + m.OCIMachine.Status.CreateBackendWorkRequestId = *resp.OpcWorkRequestId + logger.Info("Add instance to LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) + logger.Info("Waiting for LB work request to be complete") + _, err = ociutil.AwaitLBWorkRequest(ctx, m.LoadBalancerClient, resp.OpcWorkRequestId) + if err != nil { + return err + } } } @@ -673,35 +681,43 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { if err != nil { return err } - backendSet := lb.BackendSets[APIServerLBBackendSetName] - if !m.containsNLBBackend(backendSet, m.Name()) { - logger := m.Logger.WithValues("backend-set", *backendSet.Name) - logger.Info("Checking work request status for create backend") - // we always try to create the backend if it exists during a reconcile loop and wait for the work request - // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the - // work request, the create backend call may throw an error which is ok, as reconcile will go into - // an exponential backoff - resp, err := m.NetworkLoadBalancerClient.CreateBackend(ctx, networkloadbalancer.CreateBackendRequest{ - NetworkLoadBalancerId: loadbalancerId, - BackendSetName: backendSet.Name, - CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ - IpAddress: common.String(instanceIp), - Port: common.Int(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)), - Name: common.String(m.Name()), - }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", string(m.OCIMachine.UID)), - }) - if err != nil { - return err + for _, backendSetName := range m.desiredAPIServerBackendSetNames() { + backendSet, ok := lb.BackendSets[backendSetName] + if !ok { + return errors.Errorf("backend set %q not found on network load balancer", backendSetName) } - m.OCIMachine.Status.CreateBackendWorkRequestId = *resp.OpcWorkRequestId - logger.Info("Add instance to NLB backend-set", "WorkRequestId", resp.OpcWorkRequestId) - logger.Info("Waiting for NLB work request to be complete") - _, err = ociutil.AwaitNLBWorkRequest(ctx, m.NetworkLoadBalancerClient, resp.OpcWorkRequestId) - if err != nil { - return err + if backendSet.Name == nil { + backendSet.Name = common.String(backendSetName) + } + if !m.containsNLBBackend(backendSet, m.Name()) { + logger := m.Logger.WithValues("backend-set", *backendSet.Name) + logger.Info("Checking work request status for create backend") + // we always try to create the backend if it exists during a reconcile loop and wait for the work request + // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the + // work request, the create backend call may throw an error which is ok, as reconcile will go into + // an exponential backoff + resp, err := m.NetworkLoadBalancerClient.CreateBackend(ctx, networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: loadbalancerId, + BackendSetName: backendSet.Name, + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String(instanceIp), + Port: common.Int(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)), + Name: common.String(m.Name()), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", string(m.OCIMachine.UID)), + }) + if err != nil { + return err + } + m.OCIMachine.Status.CreateBackendWorkRequestId = *resp.OpcWorkRequestId + logger.Info("Add instance to NLB backend-set", "WorkRequestId", resp.OpcWorkRequestId) + logger.Info("Waiting for NLB work request to be complete") + _, err = ociutil.AwaitNLBWorkRequest(ctx, m.NetworkLoadBalancerClient, resp.OpcWorkRequestId) + if err != nil { + return err + } + logger.Info("NLB Backend addition work request is complete") } - logger.Info("NLB Backend addition work request is complete") } } return nil @@ -729,7 +745,6 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { } return err } - backendSet := lb.BackendSets[APIServerLBBackendSetName] // in case of delete from LB backend, if the instance does not have an IP, we consider // the instance to not have been added in first place and hence return nil if len(m.OCIMachine.Status.Addresses) <= 0 { @@ -741,35 +756,40 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { return err } backendName := instanceIp + ":" + strconv.Itoa(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)) - if m.containsLBBackend(backendSet, backendName) { - logger := m.Logger.WithValues("backend-set", *backendSet.Name) - // in OCI CLI, the colon in the backend name is replaced by %3A - // replace the colon in the backend name by %3A to avoid the error in PCA - escapedBackendName := url.QueryEscape(backendName) - // we always try to delete the backend if it exists during a reconcile loop and wait for the work request - // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the - // work request, the delete backend call may throw an error which is ok, as reconcile will go into - // an exponential backoff - resp, err := m.LoadBalancerClient.DeleteBackend(ctx, loadbalancer.DeleteBackendRequest{ - LoadBalancerId: loadbalancerId, - BackendSetName: backendSet.Name, - BackendName: common.String(escapedBackendName), - }) - if err != nil { - logger.Error(err, "Delete instance from LB backend-set failed", - "backendSetName", *backendSet.Name, - "backendName", escapedBackendName, - ) - return err + for backendSetName, backendSet := range lb.BackendSets { + if backendSet.Name == nil { + backendSet.Name = common.String(backendSetName) } - m.OCIMachine.Status.DeleteBackendWorkRequestId = *resp.OpcWorkRequestId - logger.Info("Delete instance from LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) - logger.Info("Waiting for LB work request to be complete") - _, err = ociutil.AwaitLBWorkRequest(ctx, m.LoadBalancerClient, resp.OpcWorkRequestId) - if err != nil { - return err + if m.containsLBBackend(backendSet, backendName) { + logger := m.Logger.WithValues("backend-set", *backendSet.Name) + // in OCI CLI, the colon in the backend name is replaced by %3A + // replace the colon in the backend name by %3A to avoid the error in PCA + escapedBackendName := url.QueryEscape(backendName) + // we always try to delete the backend if it exists during a reconcile loop and wait for the work request + // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the + // work request, the delete backend call may throw an error which is ok, as reconcile will go into + // an exponential backoff + resp, err := m.LoadBalancerClient.DeleteBackend(ctx, loadbalancer.DeleteBackendRequest{ + LoadBalancerId: loadbalancerId, + BackendSetName: backendSet.Name, + BackendName: common.String(escapedBackendName), + }) + if err != nil { + logger.Error(err, "Delete instance from LB backend-set failed", + "backendSetName", *backendSet.Name, + "backendName", escapedBackendName, + ) + return err + } + m.OCIMachine.Status.DeleteBackendWorkRequestId = *resp.OpcWorkRequestId + logger.Info("Delete instance from LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) + logger.Info("Waiting for LB work request to be complete") + _, err = ociutil.AwaitLBWorkRequest(ctx, m.LoadBalancerClient, resp.OpcWorkRequestId) + if err != nil { + return err + } + logger.Info("LB Backend addition work request is complete") } - logger.Info("LB Backend addition work request is complete") } } else { lb, err := m.NetworkLoadBalancerClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ @@ -782,33 +802,46 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { } return err } - backendSet := lb.BackendSets[APIServerLBBackendSetName] - if m.containsNLBBackend(backendSet, m.Name()) { - logger := m.Logger.WithValues("backend-set", *backendSet.Name) - // we always try to delete the backend if it exists during a reconcile loop and wait for the work request - // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the - // work request, the delete backend call may throw an error which is ok, as reconcile will go into - // an exponential backoff - resp, err := m.NetworkLoadBalancerClient.DeleteBackend(ctx, networkloadbalancer.DeleteBackendRequest{ - NetworkLoadBalancerId: loadbalancerId, - BackendSetName: backendSet.Name, - BackendName: common.String(m.Name()), - }) - if err != nil { - return err + for backendSetName, backendSet := range lb.BackendSets { + if backendSet.Name == nil { + backendSet.Name = common.String(backendSetName) } - m.OCIMachine.Status.DeleteBackendWorkRequestId = *resp.OpcWorkRequestId - logger.Info("Delete instance from LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) - logger.Info("Waiting for LB work request to be complete") - _, err = ociutil.AwaitNLBWorkRequest(ctx, m.NetworkLoadBalancerClient, resp.OpcWorkRequestId) - if err != nil { - return err + if m.containsNLBBackend(backendSet, m.Name()) { + logger := m.Logger.WithValues("backend-set", *backendSet.Name) + // we always try to delete the backend if it exists during a reconcile loop and wait for the work request + // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the + // work request, the delete backend call may throw an error which is ok, as reconcile will go into + // an exponential backoff + resp, err := m.NetworkLoadBalancerClient.DeleteBackend(ctx, networkloadbalancer.DeleteBackendRequest{ + NetworkLoadBalancerId: loadbalancerId, + BackendSetName: backendSet.Name, + BackendName: common.String(m.Name()), + }) + if err != nil { + return err + } + m.OCIMachine.Status.DeleteBackendWorkRequestId = *resp.OpcWorkRequestId + logger.Info("Delete instance from LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) + logger.Info("Waiting for LB work request to be complete") + _, err = ociutil.AwaitNLBWorkRequest(ctx, m.NetworkLoadBalancerClient, resp.OpcWorkRequestId) + if err != nil { + return err + } } } } return nil } +func (m *MachineScope) desiredAPIServerBackendSetNames() []string { + backendSets := m.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec.CanonicalBackendSets() + names := make([]string, 0, len(backendSets)) + for _, backendSet := range backendSets { + names = append(names, backendSet.Name) + } + return names +} + func (m *MachineScope) containsNLBBackend(backendSet networkloadbalancer.BackendSet, backendName string) bool { for _, backend := range backendSet.Backends { if ptr.StringEquals(backend.Name, backendName) { diff --git a/cloud/scope/machine_backendsets_test.go b/cloud/scope/machine_backendsets_test.go new file mode 100644 index 000000000..f57446c01 --- /dev/null +++ b/cloud/scope/machine_backendsets_test.go @@ -0,0 +1,199 @@ +package scope + +import ( + "context" + "testing" + + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer/mock_nlb" + "github.com/oracle/cluster-api-provider-oci/cloud/services/workrequests/mock_workrequests" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestNLBCreateMembershipAcrossBackendSets(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + nlbClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + wrClient := mock_workrequests.NewMockClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{UID: "uid"}, + Spec: infrastructurev1beta2.OCIClusterSpec{ + NetworkSpec: infrastructurev1beta2.NetworkSpec{ + APIServerLB: infrastructurev1beta2.LoadBalancer{ + LoadBalancerId: common.String("nlbid"), + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b"}, + }, + }, + }, + }, + }, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + + ms, err := NewMachineScope(MachineScopeParams{ + NetworkLoadBalancerClient: nlbClient, + WorkRequestsClient: wrClient, + OCIMachine: &infrastructurev1beta2.OCIMachine{ + ObjectMeta: metav1.ObjectMeta{Name: "test", UID: "uid"}, + }, + Machine: &clusterv1.Machine{}, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{ + OCICluster: ociCluster, + }, + Client: client, + }) + g.Expect(err).To(BeNil()) + ms.OCIMachine.Status.Addresses = []clusterv1beta1.MachineAddress{{ + Type: clusterv1beta1.MachineInternalIP, + Address: "1.1.1.1", + }} + + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + BackendSets: map[string]networkloadbalancer.BackendSet{ + "set-a": {Name: common.String("set-a"), Backends: []networkloadbalancer.Backend{}}, + "set-b": {Name: common.String("set-b"), Backends: []networkloadbalancer.Backend{}}, + }, + }, + }, nil) + + nlbClient.EXPECT().CreateBackend(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + BackendSetName: common.String("set-a"), + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String("1.1.1.1"), + Port: common.Int(6443), + Name: common.String("test"), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", "uid"), + })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-a")}, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wrid-a"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil) + + nlbClient.EXPECT().CreateBackend(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + BackendSetName: common.String("set-b"), + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String("1.1.1.1"), + Port: common.Int(6443), + Name: common.String("test"), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", "uid"), + })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-b")}, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wrid-b"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil) + + g.Expect(ms.ReconcileCreateInstanceOnLB(context.Background())).To(Succeed()) +} + +func TestNLBDeleteMembershipFromAllBackendSets(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + nlbClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + wrClient := mock_workrequests.NewMockClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{UID: "uid"}, + Spec: infrastructurev1beta2.OCIClusterSpec{ + NetworkSpec: infrastructurev1beta2.NetworkSpec{ + APIServerLB: infrastructurev1beta2.LoadBalancer{ + LoadBalancerId: common.String("nlbid"), + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: "set-a"}, + }, + }, + }, + }, + }, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + + ms, err := NewMachineScope(MachineScopeParams{ + NetworkLoadBalancerClient: nlbClient, + WorkRequestsClient: wrClient, + OCIMachine: &infrastructurev1beta2.OCIMachine{ + ObjectMeta: metav1.ObjectMeta{Name: "test", UID: "uid"}, + }, + Machine: &clusterv1.Machine{}, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{ + OCICluster: ociCluster, + }, + Client: client, + }) + g.Expect(err).To(BeNil()) + + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + BackendSets: map[string]networkloadbalancer.BackendSet{ + "set-a": {Name: common.String("set-a"), Backends: []networkloadbalancer.Backend{{Name: common.String("test")}}}, + "set-old": {Name: common.String("set-old"), Backends: []networkloadbalancer.Backend{{Name: common.String("test")}}}, + }, + }, + }, nil) + + seen := map[string]bool{} + nlbClient.EXPECT().DeleteBackend(gomock.Any(), gomock.Any()).Times(2).DoAndReturn( + func(_ context.Context, req networkloadbalancer.DeleteBackendRequest) (networkloadbalancer.DeleteBackendResponse, error) { + if req.NetworkLoadBalancerId == nil || *req.NetworkLoadBalancerId != "nlbid" { + t.Fatalf("unexpected load balancer id: %#v", req.NetworkLoadBalancerId) + } + if req.BackendName == nil || *req.BackendName != "test" { + t.Fatalf("unexpected backend name: %#v", req.BackendName) + } + if req.BackendSetName == nil { + t.Fatalf("backend set name is nil") + } + name := *req.BackendSetName + if name != "set-a" && name != "set-old" { + t.Fatalf("unexpected backend set name: %q", name) + } + seen[name] = true + return networkloadbalancer.DeleteBackendResponse{ + OpcWorkRequestId: common.String("wrid-" + name), + }, nil + }, + ) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Any()).Times(2).DoAndReturn( + func(_ context.Context, req networkloadbalancer.GetWorkRequestRequest) (networkloadbalancer.GetWorkRequestResponse, error) { + if req.WorkRequestId == nil { + t.Fatalf("work request id is nil") + } + return networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil + }, + ) + + g.Expect(ms.ReconcileDeleteInstanceOnLB(context.Background())).To(Succeed()) + g.Expect(seen["set-a"]).To(BeTrue()) + g.Expect(seen["set-old"]).To(BeTrue()) +} From de97991fba4026e7e28c2dbe5a62269072f987d4 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 3 Mar 2026 14:43:09 -0500 Subject: [PATCH 06/31] feat(scope): reconcile lb listener/backend-set resources on updates --- cloud/scope/load_balancer_reconciler.go | 144 ++++++++++++ .../scope/network_load_balancer_reconciler.go | 206 ++++++++++++++++++ 2 files changed, 350 insertions(+) diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index b6babc3af..faf20ce23 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -29,6 +29,15 @@ import ( clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" ) +type lbExtendedClient interface { + CreateBackendSet(ctx context.Context, request loadbalancer.CreateBackendSetRequest) (response loadbalancer.CreateBackendSetResponse, err error) + UpdateBackendSet(ctx context.Context, request loadbalancer.UpdateBackendSetRequest) (response loadbalancer.UpdateBackendSetResponse, err error) + DeleteBackendSet(ctx context.Context, request loadbalancer.DeleteBackendSetRequest) (response loadbalancer.DeleteBackendSetResponse, err error) + CreateListener(ctx context.Context, request loadbalancer.CreateListenerRequest) (response loadbalancer.CreateListenerResponse, err error) + UpdateListener(ctx context.Context, request loadbalancer.UpdateListenerRequest) (response loadbalancer.UpdateListenerResponse, err error) + DeleteListener(ctx context.Context, request loadbalancer.DeleteListenerRequest) (response loadbalancer.DeleteListenerResponse, err error) +} + // ReconcileApiServerLB tries to move the Load Balancer to the desired OCICluster Spec func (s *ClusterScope) ReconcileApiServerLB(ctx context.Context) error { desiredApiServerLb := s.LBSpec() @@ -135,6 +144,141 @@ func (s *ClusterScope) UpdateLB(ctx context.Context, lb infrastructurev1beta2.Lo s.Logger.Error(err, "failed to reconcile the apiserver LB, failed to update lb") return errors.Wrap(err, "failed to reconcile the apiserver LB, failed to update lb") } + if err := s.reconcileLBResources(ctx, lb); err != nil { + return err + } + return nil +} + +func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructurev1beta2.LoadBalancer) error { + extendedClient, ok := any(s.LoadBalancerClient).(lbExtendedClient) + if !ok { + s.Logger.Info("LB client does not support extended listener/backend-set reconcile") + return nil + } + lbID := s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.LoadBalancerId + actualResp, err := s.LoadBalancerClient.GetLoadBalancer(ctx, loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: lbID, + }) + if err != nil { + return errors.Wrap(err, "failed to get lb for resource reconciliation") + } + desiredListeners, desiredBackendSets := s.buildDesiredLBListenersAndBackendSets(lb) + + for name, desired := range desiredListeners { + actual, exists := actualResp.LoadBalancer.Listeners[name] + if !exists { + resp, err := extendedClient.CreateListener(ctx, loadbalancer.CreateListenerRequest{ + LoadBalancerId: lbID, + CreateListenerDetails: loadbalancer.CreateListenerDetails{ + Name: common.String(name), + DefaultBackendSetName: desired.DefaultBackendSetName, + Port: desired.Port, + Protocol: desired.Protocol, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to create lb listener %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting create listener %q", name) + } + continue + } + if !intPtrEqual(actual.Port, desired.Port) || + !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || + !ptr.StringEquals(actual.Protocol, ptr.ToString(desired.Protocol)) { + resp, err := extendedClient.UpdateListener(ctx, loadbalancer.UpdateListenerRequest{ + LoadBalancerId: lbID, + ListenerName: common.String(name), + UpdateListenerDetails: loadbalancer.UpdateListenerDetails{ + DefaultBackendSetName: desired.DefaultBackendSetName, + Port: desired.Port, + Protocol: desired.Protocol, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update lb listener %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update listener %q", name) + } + } + } + for name := range actualResp.LoadBalancer.Listeners { + if _, keep := desiredListeners[name]; keep { + continue + } + resp, err := extendedClient.DeleteListener(ctx, loadbalancer.DeleteListenerRequest{ + LoadBalancerId: lbID, + ListenerName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale lb listener %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete listener %q", name) + } + } + + for name, desired := range desiredBackendSets { + actual, exists := actualResp.LoadBalancer.BackendSets[name] + if !exists { + resp, err := extendedClient.CreateBackendSet(ctx, loadbalancer.CreateBackendSetRequest{ + LoadBalancerId: lbID, + CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ + Name: common.String(name), + Policy: desired.Policy, + HealthChecker: desired.HealthChecker, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to create lb backend set %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting create backend set %q", name) + } + continue + } + if !ptr.StringEquals(actual.Policy, ptr.ToString(desired.Policy)) || + actual.HealthChecker == nil || desired.HealthChecker == nil || + !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || + !ptr.StringEquals(actual.HealthChecker.Protocol, ptr.ToString(desired.HealthChecker.Protocol)) { + resp, err := extendedClient.UpdateBackendSet(ctx, loadbalancer.UpdateBackendSetRequest{ + LoadBalancerId: lbID, + BackendSetName: common.String(name), + UpdateBackendSetDetails: loadbalancer.UpdateBackendSetDetails{ + Policy: desired.Policy, + Backends: []loadbalancer.BackendDetails{}, + HealthChecker: desired.HealthChecker, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update lb backend set %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update backend set %q", name) + } + } + } + for name, actual := range actualResp.LoadBalancer.BackendSets { + if _, keep := desiredBackendSets[name]; keep { + continue + } + if len(actual.Backends) > 0 { + continue + } + resp, err := extendedClient.DeleteBackendSet(ctx, loadbalancer.DeleteBackendSetRequest{ + LoadBalancerId: lbID, + BackendSetName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale lb backend set %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete backend set %q", name) + } + } return nil } diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index 04e600fea..50854eb11 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -29,6 +29,15 @@ import ( clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" ) +type nlbExtendedClient interface { + CreateBackendSet(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (response networkloadbalancer.CreateBackendSetResponse, err error) + UpdateBackendSet(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (response networkloadbalancer.UpdateBackendSetResponse, err error) + DeleteBackendSet(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (response networkloadbalancer.DeleteBackendSetResponse, err error) + CreateListener(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (response networkloadbalancer.CreateListenerResponse, err error) + UpdateListener(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (response networkloadbalancer.UpdateListenerResponse, err error) + DeleteListener(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (response networkloadbalancer.DeleteListenerResponse, err error) +} + // ReconcileApiServerNLB tries to move the Network Load Balancer to the desired OCICluster Spec func (s *ClusterScope) ReconcileApiServerNLB(ctx context.Context) error { desiredApiServerNLB := s.NLBSpec() @@ -134,6 +143,171 @@ func (s *ClusterScope) UpdateNLB(ctx context.Context, nlb infrastructurev1beta2. s.Logger.Error(err, "failed to reconcile the apiserver NLB, failed to update nlb") return errors.Wrap(err, "failed to reconcile the apiserver NLB, failed to update nlb") } + if err := s.reconcileNLBResources(ctx, nlb); err != nil { + return err + } + return nil +} + +func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastructurev1beta2.LoadBalancer) error { + extendedClient, ok := any(s.NetworkLoadBalancerClient).(nlbExtendedClient) + if !ok { + // Unit tests use minimal mocks without listener/backend-set management methods. + s.Logger.Info("NLB client does not support extended listener/backend-set reconcile") + return nil + } + nlbID := s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.LoadBalancerId + actualResp, err := s.NetworkLoadBalancerClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: nlbID, + }) + if err != nil { + return errors.Wrap(err, "failed to get nlb for resource reconciliation") + } + desiredListeners, desiredBackendSets := s.buildDesiredNLBListenersAndBackendSets(nlb) + + for name, desired := range desiredListeners { + actual, exists := actualResp.NetworkLoadBalancer.Listeners[name] + if !exists { + resp, err := extendedClient.CreateListener(ctx, networkloadbalancer.CreateListenerRequest{ + NetworkLoadBalancerId: nlbID, + CreateListenerDetails: networkloadbalancer.CreateListenerDetails{ + Name: common.String(name), + DefaultBackendSetName: desired.DefaultBackendSetName, + Port: desired.Port, + Protocol: desired.Protocol, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to create nlb listener %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting create listener %q", name) + } + continue + } + if !intPtrEqual(actual.Port, desired.Port) || + !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || + actual.Protocol != desired.Protocol { + resp, err := extendedClient.UpdateListener(ctx, networkloadbalancer.UpdateListenerRequest{ + NetworkLoadBalancerId: nlbID, + ListenerName: common.String(name), + UpdateListenerDetails: networkloadbalancer.UpdateListenerDetails{ + DefaultBackendSetName: desired.DefaultBackendSetName, + Port: desired.Port, + Protocol: desired.Protocol, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update nlb listener %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update listener %q", name) + } + } + } + for name := range actualResp.NetworkLoadBalancer.Listeners { + if _, keep := desiredListeners[name]; keep { + continue + } + resp, err := extendedClient.DeleteListener(ctx, networkloadbalancer.DeleteListenerRequest{ + NetworkLoadBalancerId: nlbID, + ListenerName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale nlb listener %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete listener %q", name) + } + } + + for name, desired := range desiredBackendSets { + actual, exists := actualResp.NetworkLoadBalancer.BackendSets[name] + if !exists { + resp, err := extendedClient.CreateBackendSet(ctx, networkloadbalancer.CreateBackendSetRequest{ + NetworkLoadBalancerId: nlbID, + CreateBackendSetDetails: networkloadbalancer.CreateBackendSetDetails{ + Name: common.String(name), + Policy: desired.Policy, + HealthChecker: nlbHealthCheckerToDetails(desired.HealthChecker), + IsPreserveSource: desired.IsPreserveSource, + IsFailOpen: desired.IsFailOpen, + IsInstantFailoverEnabled: desired.IsInstantFailoverEnabled, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to create nlb backend set %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting create backend set %q", name) + } + continue + } + if actual.HealthChecker == nil || desired.HealthChecker == nil { + // Reconcile nullable health checker by updating if either side is nil. + resp, err := extendedClient.UpdateBackendSet(ctx, networkloadbalancer.UpdateBackendSetRequest{ + NetworkLoadBalancerId: nlbID, + BackendSetName: common.String(name), + UpdateBackendSetDetails: networkloadbalancer.UpdateBackendSetDetails{ + Policy: common.String(string(desired.Policy)), + HealthChecker: nlbHealthCheckerToDetails(desired.HealthChecker), + IsPreserveSource: desired.IsPreserveSource, + IsFailOpen: desired.IsFailOpen, + IsInstantFailoverEnabled: desired.IsInstantFailoverEnabled, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update nlb backend set %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update backend set %q", name) + } + continue + } + if actual.Policy != desired.Policy || + !boolPtrEqual(actual.IsPreserveSource, desired.IsPreserveSource) || + !boolPtrEqual(actual.IsFailOpen, desired.IsFailOpen) || + !boolPtrEqual(actual.IsInstantFailoverEnabled, desired.IsInstantFailoverEnabled) || + !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || + actual.HealthChecker.Protocol != desired.HealthChecker.Protocol || + !ptr.StringEquals(actual.HealthChecker.UrlPath, ptr.ToString(desired.HealthChecker.UrlPath)) { + resp, err := extendedClient.UpdateBackendSet(ctx, networkloadbalancer.UpdateBackendSetRequest{ + NetworkLoadBalancerId: nlbID, + BackendSetName: common.String(name), + UpdateBackendSetDetails: networkloadbalancer.UpdateBackendSetDetails{ + Policy: common.String(string(desired.Policy)), + HealthChecker: nlbHealthCheckerToDetails(desired.HealthChecker), + IsPreserveSource: desired.IsPreserveSource, + IsFailOpen: desired.IsFailOpen, + IsInstantFailoverEnabled: desired.IsInstantFailoverEnabled, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update nlb backend set %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update backend set %q", name) + } + } + } + for name, actual := range actualResp.NetworkLoadBalancer.BackendSets { + if _, keep := desiredBackendSets[name]; keep { + continue + } + if len(actual.Backends) > 0 { + continue + } + resp, err := extendedClient.DeleteBackendSet(ctx, networkloadbalancer.DeleteBackendSetRequest{ + NetworkLoadBalancerId: nlbID, + BackendSetName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale nlb backend set %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete backend set %q", name) + } + } return nil } @@ -264,6 +438,38 @@ func (s *ClusterScope) buildDesiredNLBListenersAndBackendSets(lb infrastructurev return listenerDetails, backendSetDetails } +func boolPtrEqual(a, b *bool) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +func intPtrEqual(a, b *int) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +func nlbHealthCheckerToDetails(in *networkloadbalancer.HealthChecker) *networkloadbalancer.HealthCheckerDetails { + if in == nil { + return nil + } + return &networkloadbalancer.HealthCheckerDetails{ + Protocol: in.Protocol, + Port: in.Port, + Retries: in.Retries, + TimeoutInMillis: in.TimeoutInMillis, + IntervalInMillis: in.IntervalInMillis, + UrlPath: in.UrlPath, + ResponseBodyRegex: in.ResponseBodyRegex, + ReturnCode: in.ReturnCode, + RequestData: in.RequestData, + ResponseData: in.ResponseData, + } +} + func (s *ClusterScope) getNetworkLoadbalancerIp(nlb networkloadbalancer.NetworkLoadBalancer) (*string, error) { var nlbIp *string if len(nlb.IpAddresses) < 1 { From 50fd7aebbdfeea554a2ca20dc597dd33dbd71681 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 3 Mar 2026 14:49:26 -0500 Subject: [PATCH 07/31] test(scope): cover deterministic listener naming and port selection --- cloud/scope/apiserver_listener_naming_test.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 cloud/scope/apiserver_listener_naming_test.go diff --git a/cloud/scope/apiserver_listener_naming_test.go b/cloud/scope/apiserver_listener_naming_test.go new file mode 100644 index 000000000..c3f0c9a6d --- /dev/null +++ b/cloud/scope/apiserver_listener_naming_test.go @@ -0,0 +1,39 @@ +package scope + +import ( + "testing" + + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" +) + +func TestDesiredAPIServerListenerName(t *testing.T) { + t.Run("keeps legacy listener name for single legacy backend set", func(t *testing.T) { + got := desiredAPIServerListenerName(0, 1, APIServerLBBackendSetName) + if got != APIServerLBListener { + t.Fatalf("expected legacy listener name %q, got %q", APIServerLBListener, got) + } + }) + + t.Run("uses stable hashed names for secondary listeners", func(t *testing.T) { + a := desiredAPIServerListenerName(1, 2, "rollout-set") + b := desiredAPIServerListenerName(1, 2, "rollout-set") + c := desiredAPIServerListenerName(1, 2, "another-set") + if a != b { + t.Fatalf("expected deterministic name for same backend set, got %q and %q", a, b) + } + if a == c { + t.Fatalf("expected different names for different backend sets, got %q and %q", a, c) + } + }) +} + +func TestDesiredAPIServerListenerPort(t *testing.T) { + defaultPort := int32(6443) + if got := desiredAPIServerListenerPort(defaultPort, infrastructurev1beta2.NLBBackendSet{}); got != defaultPort { + t.Fatalf("expected default port %d, got %d", defaultPort, got) + } + customPort := int32(9345) + if got := desiredAPIServerListenerPort(defaultPort, infrastructurev1beta2.NLBBackendSet{ListenerPort: &customPort}); got != customPort { + t.Fatalf("expected custom port %d, got %d", customPort, got) + } +} From 41f61a45429e627ceb94b7cbd6dcc8eb31bc3fb0 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 3 Mar 2026 14:50:40 -0500 Subject: [PATCH 08/31] docs: add multi-backend-set api server lb migration guidance --- docs/src/networking/custom-networking.md | 74 ++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/docs/src/networking/custom-networking.md b/docs/src/networking/custom-networking.md index 83cceb742..0b0cf1c72 100644 --- a/docs/src/networking/custom-networking.md +++ b/docs/src/networking/custom-networking.md @@ -300,6 +300,80 @@ spec: loadBalancerType: "lb" ``` +## Configure multiple API-server backend sets and listeners + +CAPOCI supports multiple API-server backend sets through `networkSpec.apiServerLoadBalancer.nlbSpec.backendSets`. +Each backend set can optionally define a dedicated `listenerPort`. + +Field precedence and compatibility rules: + +1. `backendSets` is the canonical field. +2. Legacy `backendSetDetails` remains supported for backward compatibility. +3. If both are set, `backendSets` takes precedence and validation rejects mixed usage. +4. If only legacy `backendSetDetails` is set, CAPOCI preserves single-backend-set behavior. + +Example (NLB default) with multiple listeners: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + name: "${CLUSTER_NAME}" +spec: + compartmentId: "${OCI_COMPARTMENT_ID}" + networkSpec: + apiServerLoadBalancer: + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: rke2-registration + listenerPort: 9345 + backendSetDetails: + healthChecker: + urlPath: /v1-rke2/readyz +``` + +Equivalent configuration for OCI LBaaS is supported by setting `loadBalancerType: "lb"`: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + name: "${CLUSTER_NAME}" +spec: + compartmentId: "${OCI_COMPARTMENT_ID}" + networkSpec: + apiServerLoadBalancer: + loadBalancerType: "lb" + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + - name: rke2-registration + listenerPort: 9345 +``` + +### Migration from legacy `backendSetDetails` + +Use this sequence: + +1. Keep your existing legacy `backendSetDetails` (no behavior change). +2. Replace it with canonical `backendSets` and include the default set (`apiserver-lb-backendset`) first. +3. Add additional backend sets with unique names and optional `listenerPort`. +4. Apply manifests and wait for reconcile to create/update listeners and backend sets. +5. During rollout, old/new backend sets can coexist; machines are reconciled into active sets. +6. After transition, remove no-longer-needed backend sets from `backendSets`. CAPOCI garbage-collects stale listeners and stale backend sets when safe (empty/unreferenced). + +### Rollout semantics and caveats + +- The first canonical backend set remains the primary API-server listener target (`apiserver-lb-listener`). +- Additional backend sets create additional deterministic listeners for management endpoints. +- `listenerPort` values must be unique and within `1-65535`. +- Backend set names must be unique, and match provider constraints (alphanumeric, `-`, `_`). +- CAPOCI does not require immediate manual migration for old objects; legacy manifests continue to decode. + ## Example spec to use custom role CAPOCI can be used to create Subnet/NSG in the VCN for custom workloads such as private load balancers, From 404d91a34b379e1c3acb605ef4ff1cee5833ddbb Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Thu, 5 Mar 2026 15:34:34 -0500 Subject: [PATCH 09/31] feat (fix): set backend for second backend set --- Makefile | 2 + cloud/scope/machine.go | 87 ++++++++-- cloud/scope/machine_backendsets_test.go | 103 +++++++++++- .../cluster-template-multiple-backends.yaml | 153 ++++++++++++++++++ test/e2e/cluster_test.go | 44 +++++ test/e2e/config/e2e_conf.yaml | 2 + .../cluster.yaml | 20 +++ .../kustomization.yaml | 7 + .../cluster.yaml | 19 +++ .../kustomization.yaml | 7 + 10 files changed, 424 insertions(+), 20 deletions(-) create mode 100644 templates/cluster-template-multiple-backends.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/kustomization.yaml diff --git a/Makefile b/Makefile index f66b1a23f..45c8d56e7 100644 --- a/Makefile +++ b/Makefile @@ -287,6 +287,8 @@ generate-e2e-templates: $(KUSTOMIZE) $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-custom-networking-seclist --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-custom-networking-seclist.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-custom-networking-nsg --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-custom-networking-nsg.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-node-nsg --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-node-nsg.yaml + $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-nlb --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-nlb.yaml + $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-lb --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-lb.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-cluster-class --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-cluster-class.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-local-vcn-peering --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-local-vcn-peering.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-remote-vcn-peering --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-remote-vcn-peering.yaml diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index bed3a5ca8..46ae67b47 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -634,9 +634,6 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { if err != nil { return err } - // When creating a LB, there is no way to set the backend Name, default backend name is the instance IP and port - // So we use default backend name instead of machine name - backendName := instanceIp + ":" + strconv.Itoa(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)) for _, backendSetName := range m.desiredAPIServerBackendSetNames() { backendSet, ok := lb.BackendSets[backendSetName] if !ok { @@ -645,6 +642,9 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { if backendSet.Name == nil { backendSet.Name = common.String(backendSetName) } + backendPort := m.desiredAPIServerBackendPort(backendSetName) + // For LBaaS backend names are represented as ":". + backendName := instanceIp + ":" + strconv.Itoa(int(backendPort)) if !m.containsLBBackend(backendSet, backendName) { logger := m.Logger.WithValues("backend-set", *backendSet.Name) logger.Info("Checking work request status for create backend") @@ -657,9 +657,9 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { BackendSetName: backendSet.Name, CreateBackendDetails: loadbalancer.CreateBackendDetails{ IpAddress: common.String(instanceIp), - Port: common.Int(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)), + Port: common.Int(int(backendPort)), }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", string(m.OCIMachine.UID)), + OpcRetryToken: m.createBackendRetryToken(backendSetName, backendPort), }) if err != nil { return err @@ -681,7 +681,8 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { if err != nil { return err } - for _, backendSetName := range m.desiredAPIServerBackendSetNames() { + desiredBackendSetNames := m.desiredAPIServerBackendSetNames() + for _, backendSetName := range desiredBackendSetNames { backendSet, ok := lb.BackendSets[backendSetName] if !ok { return errors.Errorf("backend set %q not found on network load balancer", backendSetName) @@ -689,7 +690,10 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { if backendSet.Name == nil { backendSet.Name = common.String(backendSetName) } - if !m.containsNLBBackend(backendSet, m.Name()) { + desiredBackendName := m.desiredNLBBackendName(backendSetName, desiredBackendSetNames) + legacyBackendName := m.Name() + backendPort := m.desiredAPIServerBackendPort(backendSetName) + if !m.containsNLBBackend(backendSet, desiredBackendName) && !m.containsNLBBackend(backendSet, legacyBackendName) { logger := m.Logger.WithValues("backend-set", *backendSet.Name) logger.Info("Checking work request status for create backend") // we always try to create the backend if it exists during a reconcile loop and wait for the work request @@ -701,10 +705,10 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { BackendSetName: backendSet.Name, CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ IpAddress: common.String(instanceIp), - Port: common.Int(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)), - Name: common.String(m.Name()), + Port: common.Int(int(backendPort)), + Name: common.String(desiredBackendName), }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", string(m.OCIMachine.UID)), + OpcRetryToken: m.createBackendRetryToken(backendSetName, backendPort), }) if err != nil { return err @@ -755,16 +759,23 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { if err != nil { return err } - backendName := instanceIp + ":" + strconv.Itoa(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)) for backendSetName, backendSet := range lb.BackendSets { if backendSet.Name == nil { backendSet.Name = common.String(backendSetName) } - if m.containsLBBackend(backendSet, backendName) { + desiredBackendName := m.desiredLBBackendName(instanceIp, backendSetName) + legacyBackendName := instanceIp + ":" + strconv.Itoa(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)) + backendNameToDelete := "" + if m.containsLBBackend(backendSet, desiredBackendName) { + backendNameToDelete = desiredBackendName + } else if desiredBackendName != legacyBackendName && m.containsLBBackend(backendSet, legacyBackendName) { + backendNameToDelete = legacyBackendName + } + if backendNameToDelete != "" { logger := m.Logger.WithValues("backend-set", *backendSet.Name) // in OCI CLI, the colon in the backend name is replaced by %3A // replace the colon in the backend name by %3A to avoid the error in PCA - escapedBackendName := url.QueryEscape(backendName) + escapedBackendName := url.QueryEscape(backendNameToDelete) // we always try to delete the backend if it exists during a reconcile loop and wait for the work request // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the // work request, the delete backend call may throw an error which is ok, as reconcile will go into @@ -802,11 +813,20 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { } return err } + desiredBackendSetNames := m.desiredAPIServerBackendSetNames() for backendSetName, backendSet := range lb.BackendSets { if backendSet.Name == nil { backendSet.Name = common.String(backendSetName) } - if m.containsNLBBackend(backendSet, m.Name()) { + desiredBackendName := m.desiredNLBBackendName(backendSetName, desiredBackendSetNames) + legacyBackendName := m.Name() + backendNameToDelete := "" + if m.containsNLBBackend(backendSet, desiredBackendName) { + backendNameToDelete = desiredBackendName + } else if desiredBackendName != legacyBackendName && m.containsNLBBackend(backendSet, legacyBackendName) { + backendNameToDelete = legacyBackendName + } + if backendNameToDelete != "" { logger := m.Logger.WithValues("backend-set", *backendSet.Name) // we always try to delete the backend if it exists during a reconcile loop and wait for the work request // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the @@ -815,7 +835,7 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { resp, err := m.NetworkLoadBalancerClient.DeleteBackend(ctx, networkloadbalancer.DeleteBackendRequest{ NetworkLoadBalancerId: loadbalancerId, BackendSetName: backendSet.Name, - BackendName: common.String(m.Name()), + BackendName: common.String(backendNameToDelete), }) if err != nil { return err @@ -842,6 +862,43 @@ func (m *MachineScope) desiredAPIServerBackendSetNames() []string { return names } +func (m *MachineScope) desiredAPIServerBackendPort(backendSetName string) int32 { + defaultPort := m.OCIClusterAccessor.GetControlPlaneEndpoint().Port + backendSets := m.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec.CanonicalBackendSets() + for _, backendSet := range backendSets { + if backendSet.Name == backendSetName { + return desiredAPIServerListenerPort(defaultPort, backendSet) + } + } + return defaultPort +} + +func (m *MachineScope) createBackendRetryToken(backendSetName string, backendPort int32) *string { + uid := string(m.OCIMachine.UID) + if len(m.desiredAPIServerBackendSetNames()) <= 1 { + return ociutil.GetOPCRetryToken("%s-%s", "create-backend", uid) + } + scope := fmt.Sprintf("%s:%d", backendSetName, backendPort) + return ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", uid, stableShortHash(scope)) +} + +func (m *MachineScope) desiredLBBackendName(instanceIP, backendSetName string) string { + backendPort := m.desiredAPIServerBackendPort(backendSetName) + return instanceIP + ":" + strconv.Itoa(int(backendPort)) +} + +func (m *MachineScope) desiredNLBBackendName(backendSetName string, desiredBackendSetNames []string) string { + // Preserve current backend naming for all single-backend-set configurations. + if len(desiredBackendSetNames) <= 1 { + return m.Name() + } + // Preserve legacy name for the primary/default API server backend set. + if backendSetName == APIServerLBBackendSetName { + return m.Name() + } + return fmt.Sprintf("%s-%s", m.Name(), stableShortHash(backendSetName)) +} + func (m *MachineScope) containsNLBBackend(backendSet networkloadbalancer.BackendSet, backendName string) bool { for _, backend := range backendSet.Backends { if ptr.StringEquals(backend.Name, backendName) { diff --git a/cloud/scope/machine_backendsets_test.go b/cloud/scope/machine_backendsets_test.go index f57446c01..34b935c43 100644 --- a/cloud/scope/machine_backendsets_test.go +++ b/cloud/scope/machine_backendsets_test.go @@ -2,6 +2,7 @@ package scope import ( "context" + "fmt" "testing" "github.com/golang/mock/gomock" @@ -80,9 +81,9 @@ func TestNLBCreateMembershipAcrossBackendSets(t *testing.T) { CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ IpAddress: common.String("1.1.1.1"), Port: common.Int(6443), - Name: common.String("test"), + Name: common.String(fmt.Sprintf("test-%s", stableShortHash("set-a"))), }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", "uid"), + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("set-a:6443")), })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-a")}, nil) nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ WorkRequestId: common.String("wrid-a"), @@ -96,9 +97,9 @@ func TestNLBCreateMembershipAcrossBackendSets(t *testing.T) { CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ IpAddress: common.String("1.1.1.1"), Port: common.Int(6443), - Name: common.String("test"), + Name: common.String(fmt.Sprintf("test-%s", stableShortHash("set-b"))), }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", "uid"), + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("set-b:6443")), })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-b")}, nil) nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ WorkRequestId: common.String("wrid-b"), @@ -154,7 +155,7 @@ func TestNLBDeleteMembershipFromAllBackendSets(t *testing.T) { })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ BackendSets: map[string]networkloadbalancer.BackendSet{ - "set-a": {Name: common.String("set-a"), Backends: []networkloadbalancer.Backend{{Name: common.String("test")}}}, + "set-a": {Name: common.String("set-a"), Backends: []networkloadbalancer.Backend{{Name: common.String("test")}}}, "set-old": {Name: common.String("set-old"), Backends: []networkloadbalancer.Backend{{Name: common.String("test")}}}, }, }, @@ -197,3 +198,95 @@ func TestNLBDeleteMembershipFromAllBackendSets(t *testing.T) { g.Expect(seen["set-a"]).To(BeTrue()) g.Expect(seen["set-old"]).To(BeTrue()) } + +func TestNLBCreateMembership_DefaultAndSecondaryBackendSetNaming(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + nlbClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + wrClient := mock_workrequests.NewMockClient(mockCtrl) + client := fake.NewClientBuilder().Build() + secondaryPort := int32(9345) + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{UID: "uid"}, + Spec: infrastructurev1beta2.OCIClusterSpec{ + NetworkSpec: infrastructurev1beta2.NetworkSpec{ + APIServerLB: infrastructurev1beta2.LoadBalancer{ + LoadBalancerId: common.String("nlbid"), + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "apiserver-lb-backendset-2", ListenerPort: &secondaryPort}, + }, + }, + }, + }, + }, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + + ms, err := NewMachineScope(MachineScopeParams{ + NetworkLoadBalancerClient: nlbClient, + WorkRequestsClient: wrClient, + OCIMachine: &infrastructurev1beta2.OCIMachine{ + ObjectMeta: metav1.ObjectMeta{Name: "test", UID: "uid"}, + }, + Machine: &clusterv1.Machine{}, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{ + OCICluster: ociCluster, + }, + Client: client, + }) + g.Expect(err).To(BeNil()) + ms.OCIMachine.Status.Addresses = []clusterv1beta1.MachineAddress{{ + Type: clusterv1beta1.MachineInternalIP, + Address: "1.1.1.1", + }} + + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: {Name: common.String(APIServerLBBackendSetName), Backends: []networkloadbalancer.Backend{}}, + "apiserver-lb-backendset-2": {Name: common.String("apiserver-lb-backendset-2"), Backends: []networkloadbalancer.Backend{}}, + }, + }, + }, nil) + + nlbClient.EXPECT().CreateBackend(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + BackendSetName: common.String(APIServerLBBackendSetName), + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String("1.1.1.1"), + Port: common.Int(6443), + Name: common.String("test"), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash(APIServerLBBackendSetName+":6443")), + })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-default")}, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wrid-default"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil) + + nlbClient.EXPECT().CreateBackend(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + BackendSetName: common.String("apiserver-lb-backendset-2"), + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String("1.1.1.1"), + Port: common.Int(9345), + Name: common.String(fmt.Sprintf("test-%s", stableShortHash("apiserver-lb-backendset-2"))), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("apiserver-lb-backendset-2:9345")), + })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-secondary")}, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wrid-secondary"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil) + + g.Expect(ms.ReconcileCreateInstanceOnLB(context.Background())).To(Succeed()) +} diff --git a/templates/cluster-template-multiple-backends.yaml b/templates/cluster-template-multiple-backends.yaml new file mode 100644 index 000000000..b74deab90 --- /dev/null +++ b/templates/cluster-template-multiple-backends.yaml @@ -0,0 +1,153 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: OCICluster + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + compartmentId: "${OCI_COMPARTMENT_ID}" + networkSpec: + apiServerLoadBalancer: + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: apiserver-lb-backendset-2 + listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} + backendSetDetails: + healthChecker: + urlPath: /readyz +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + version: "${KUBERNETES_VERSION}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: OCIMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: ${KUBERNETES_VERSION} + apiServer: + certSANs: [localhost, 127.0.0.1] + dns: {} + etcd: {} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +kind: OCIMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + imageId: "${OCI_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_CONTROL_PLANE_MACHINE_TYPE=VM.Standard.E5.Flex}" + shapeConfig: + ocpus: "${OCI_CONTROL_PLANE_MACHINE_TYPE_OCPUS=1}" + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_CONTROL_PLANE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCIMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + imageId: "${OCI_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_NODE_MACHINE_TYPE=VM.Standard.E5.Flex}" + shapeConfig: + ocpus: "${OCI_NODE_MACHINE_TYPE_OCPUS=1}" + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${NODE_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: OCIMachineTemplate diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index 8074389bc..b5395fdba 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -156,6 +156,50 @@ var _ = Describe("Workload cluster creation", func() { validateFailureDomainSpread(namespace.Name, clusterName) }) + It("Multiple API server backend sets on NLB [PRBlocking]", func() { + clusterName = getClusterName(clusterNamePrefix, "multiple-backends-nlb") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "multiple-backends-nlb", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.MustGetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64(1), + WorkerMachineCount: pointer.Int64(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + + It("Multiple API server backend sets on LB [DailyTests]", func() { + clusterName = getClusterName(clusterNamePrefix, "multiple-backends-lb") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "multiple-backends-lb", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.MustGetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64(1), + WorkerMachineCount: pointer.Int64(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + It("Antrea as CNI - With 1 control-plane nodes and 1 worker nodes [DailyTests]", func() { clusterName = getClusterName(clusterNamePrefix, "antrea") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 7772f86ba..fff2eefbb 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -63,6 +63,8 @@ providers: - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-custom-networking-seclist.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-custom-networking-nsg.yaml" - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-multiple-node-nsg.yaml" + - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb.yaml" + - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb.yaml" - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template.yaml" - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-cluster-class.yaml" - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-cluster-class/clusterclass-test-cluster-class.yaml" diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml new file mode 100644 index 000000000..21a7aacaa --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + name: "${CLUSTER_NAME}" +spec: + networkSpec: + apiServerLoadBalancer: + loadBalancerType: "lb" + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: apiserver-lb-backendset-secondary + listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} + backendSetDetails: + healthChecker: + urlPath: /readyz diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/kustomization.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/kustomization.yaml new file mode 100644 index 000000000..a95684fe6 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/kustomization.yaml @@ -0,0 +1,7 @@ +bases: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../../bases/crs.yaml + - ../../bases/ccm.yaml +patchesStrategicMerge: + - ./cluster.yaml diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml new file mode 100644 index 000000000..dd439d36b --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + name: "${CLUSTER_NAME}" +spec: + networkSpec: + apiServerLoadBalancer: + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: apiserver-lb-backendset-secondary + listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} + backendSetDetails: + healthChecker: + urlPath: /readyz diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/kustomization.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/kustomization.yaml new file mode 100644 index 000000000..a95684fe6 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/kustomization.yaml @@ -0,0 +1,7 @@ +bases: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../../bases/crs.yaml + - ../../bases/ccm.yaml +patchesStrategicMerge: + - ./cluster.yaml From 1548a7fb681548a9a859a5d76a3952209d2de4f7 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Thu, 5 Mar 2026 16:42:56 -0500 Subject: [PATCH 10/31] feat (fix): remove lbExtendedClient and use load balancer client --- cloud/scope/load_balancer_reconciler.go | 26 ++---- cloud/services/loadbalancer/client.go | 15 +++- .../loadbalancer/mock_lb/client_mock.go | 90 +++++++++++++++++++ 3 files changed, 109 insertions(+), 22 deletions(-) diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index faf20ce23..e65b6c88e 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -29,15 +29,6 @@ import ( clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" ) -type lbExtendedClient interface { - CreateBackendSet(ctx context.Context, request loadbalancer.CreateBackendSetRequest) (response loadbalancer.CreateBackendSetResponse, err error) - UpdateBackendSet(ctx context.Context, request loadbalancer.UpdateBackendSetRequest) (response loadbalancer.UpdateBackendSetResponse, err error) - DeleteBackendSet(ctx context.Context, request loadbalancer.DeleteBackendSetRequest) (response loadbalancer.DeleteBackendSetResponse, err error) - CreateListener(ctx context.Context, request loadbalancer.CreateListenerRequest) (response loadbalancer.CreateListenerResponse, err error) - UpdateListener(ctx context.Context, request loadbalancer.UpdateListenerRequest) (response loadbalancer.UpdateListenerResponse, err error) - DeleteListener(ctx context.Context, request loadbalancer.DeleteListenerRequest) (response loadbalancer.DeleteListenerResponse, err error) -} - // ReconcileApiServerLB tries to move the Load Balancer to the desired OCICluster Spec func (s *ClusterScope) ReconcileApiServerLB(ctx context.Context) error { desiredApiServerLb := s.LBSpec() @@ -151,11 +142,6 @@ func (s *ClusterScope) UpdateLB(ctx context.Context, lb infrastructurev1beta2.Lo } func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructurev1beta2.LoadBalancer) error { - extendedClient, ok := any(s.LoadBalancerClient).(lbExtendedClient) - if !ok { - s.Logger.Info("LB client does not support extended listener/backend-set reconcile") - return nil - } lbID := s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.LoadBalancerId actualResp, err := s.LoadBalancerClient.GetLoadBalancer(ctx, loadbalancer.GetLoadBalancerRequest{ LoadBalancerId: lbID, @@ -168,7 +154,7 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu for name, desired := range desiredListeners { actual, exists := actualResp.LoadBalancer.Listeners[name] if !exists { - resp, err := extendedClient.CreateListener(ctx, loadbalancer.CreateListenerRequest{ + resp, err := s.LoadBalancerClient.CreateListener(ctx, loadbalancer.CreateListenerRequest{ LoadBalancerId: lbID, CreateListenerDetails: loadbalancer.CreateListenerDetails{ Name: common.String(name), @@ -188,7 +174,7 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu if !intPtrEqual(actual.Port, desired.Port) || !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || !ptr.StringEquals(actual.Protocol, ptr.ToString(desired.Protocol)) { - resp, err := extendedClient.UpdateListener(ctx, loadbalancer.UpdateListenerRequest{ + resp, err := s.LoadBalancerClient.UpdateListener(ctx, loadbalancer.UpdateListenerRequest{ LoadBalancerId: lbID, ListenerName: common.String(name), UpdateListenerDetails: loadbalancer.UpdateListenerDetails{ @@ -209,7 +195,7 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu if _, keep := desiredListeners[name]; keep { continue } - resp, err := extendedClient.DeleteListener(ctx, loadbalancer.DeleteListenerRequest{ + resp, err := s.LoadBalancerClient.DeleteListener(ctx, loadbalancer.DeleteListenerRequest{ LoadBalancerId: lbID, ListenerName: common.String(name), }) @@ -224,7 +210,7 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu for name, desired := range desiredBackendSets { actual, exists := actualResp.LoadBalancer.BackendSets[name] if !exists { - resp, err := extendedClient.CreateBackendSet(ctx, loadbalancer.CreateBackendSetRequest{ + resp, err := s.LoadBalancerClient.CreateBackendSet(ctx, loadbalancer.CreateBackendSetRequest{ LoadBalancerId: lbID, CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ Name: common.String(name), @@ -244,7 +230,7 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu actual.HealthChecker == nil || desired.HealthChecker == nil || !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || !ptr.StringEquals(actual.HealthChecker.Protocol, ptr.ToString(desired.HealthChecker.Protocol)) { - resp, err := extendedClient.UpdateBackendSet(ctx, loadbalancer.UpdateBackendSetRequest{ + resp, err := s.LoadBalancerClient.UpdateBackendSet(ctx, loadbalancer.UpdateBackendSetRequest{ LoadBalancerId: lbID, BackendSetName: common.String(name), UpdateBackendSetDetails: loadbalancer.UpdateBackendSetDetails{ @@ -268,7 +254,7 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu if len(actual.Backends) > 0 { continue } - resp, err := extendedClient.DeleteBackendSet(ctx, loadbalancer.DeleteBackendSetRequest{ + resp, err := s.LoadBalancerClient.DeleteBackendSet(ctx, loadbalancer.DeleteBackendSetRequest{ LoadBalancerId: lbID, BackendSetName: common.String(name), }) diff --git a/cloud/services/loadbalancer/client.go b/cloud/services/loadbalancer/client.go index a961c107e..d0dc5e958 100644 --- a/cloud/services/loadbalancer/client.go +++ b/cloud/services/loadbalancer/client.go @@ -25,10 +25,21 @@ import ( type LoadBalancerClient interface { ListLoadBalancers(ctx context.Context, request loadbalancer.ListLoadBalancersRequest) (response loadbalancer.ListLoadBalancersResponse, err error) GetLoadBalancer(ctx context.Context, request loadbalancer.GetLoadBalancerRequest) (response loadbalancer.GetLoadBalancerResponse, err error) - CreateBackend(ctx context.Context, request loadbalancer.CreateBackendRequest) (response loadbalancer.CreateBackendResponse, err error) CreateLoadBalancer(ctx context.Context, request loadbalancer.CreateLoadBalancerRequest) (response loadbalancer.CreateLoadBalancerResponse, err error) - DeleteBackend(ctx context.Context, request loadbalancer.DeleteBackendRequest) (response loadbalancer.DeleteBackendResponse, err error) GetWorkRequest(ctx context.Context, request loadbalancer.GetWorkRequestRequest) (response loadbalancer.GetWorkRequestResponse, err error) UpdateLoadBalancer(ctx context.Context, request loadbalancer.UpdateLoadBalancerRequest) (response loadbalancer.UpdateLoadBalancerResponse, err error) DeleteLoadBalancer(ctx context.Context, request loadbalancer.DeleteLoadBalancerRequest) (response loadbalancer.DeleteLoadBalancerResponse, err error) + // BackendSet + CreateBackendSet(ctx context.Context, request loadbalancer.CreateBackendSetRequest) (response loadbalancer.CreateBackendSetResponse, err error) + DeleteBackendSet(ctx context.Context, request loadbalancer.DeleteBackendSetRequest) (response loadbalancer.DeleteBackendSetResponse, err error) + UpdateBackendSet(ctx context.Context, request loadbalancer.UpdateBackendSetRequest) (response loadbalancer.UpdateBackendSetResponse, err error) + + // Backend + CreateBackend(ctx context.Context, request loadbalancer.CreateBackendRequest) (response loadbalancer.CreateBackendResponse, err error) + DeleteBackend(ctx context.Context, request loadbalancer.DeleteBackendRequest) (response loadbalancer.DeleteBackendResponse, err error) + + // Listener + CreateListener(ctx context.Context, request loadbalancer.CreateListenerRequest) (response loadbalancer.CreateListenerResponse, err error) + DeleteListener(ctx context.Context, request loadbalancer.DeleteListenerRequest) (response loadbalancer.DeleteListenerResponse, err error) + UpdateListener(ctx context.Context, request loadbalancer.UpdateListenerRequest) (response loadbalancer.UpdateListenerResponse, err error) } diff --git a/cloud/services/loadbalancer/mock_lb/client_mock.go b/cloud/services/loadbalancer/mock_lb/client_mock.go index bf9a6f7fd..ab292769d 100644 --- a/cloud/services/loadbalancer/mock_lb/client_mock.go +++ b/cloud/services/loadbalancer/mock_lb/client_mock.go @@ -50,6 +50,21 @@ func (mr *MockLoadBalancerClientMockRecorder) CreateBackend(arg0, arg1 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackend", reflect.TypeOf((*MockLoadBalancerClient)(nil).CreateBackend), arg0, arg1) } +// CreateBackendSet mocks base method. +func (m *MockLoadBalancerClient) CreateBackendSet(arg0 context.Context, arg1 loadbalancer.CreateBackendSetRequest) (loadbalancer.CreateBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateBackendSet", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.CreateBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateBackendSet indicates an expected call of CreateBackendSet. +func (mr *MockLoadBalancerClientMockRecorder) CreateBackendSet(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackendSet", reflect.TypeOf((*MockLoadBalancerClient)(nil).CreateBackendSet), arg0, arg1) +} + // CreateLoadBalancer mocks base method. func (m *MockLoadBalancerClient) CreateLoadBalancer(arg0 context.Context, arg1 loadbalancer.CreateLoadBalancerRequest) (loadbalancer.CreateLoadBalancerResponse, error) { m.ctrl.T.Helper() @@ -65,6 +80,21 @@ func (mr *MockLoadBalancerClientMockRecorder) CreateLoadBalancer(arg0, arg1 inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLoadBalancer", reflect.TypeOf((*MockLoadBalancerClient)(nil).CreateLoadBalancer), arg0, arg1) } +// CreateListener mocks base method. +func (m *MockLoadBalancerClient) CreateListener(arg0 context.Context, arg1 loadbalancer.CreateListenerRequest) (loadbalancer.CreateListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateListener", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.CreateListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateListener indicates an expected call of CreateListener. +func (mr *MockLoadBalancerClientMockRecorder) CreateListener(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateListener", reflect.TypeOf((*MockLoadBalancerClient)(nil).CreateListener), arg0, arg1) +} + // DeleteBackend mocks base method. func (m *MockLoadBalancerClient) DeleteBackend(arg0 context.Context, arg1 loadbalancer.DeleteBackendRequest) (loadbalancer.DeleteBackendResponse, error) { m.ctrl.T.Helper() @@ -80,6 +110,36 @@ func (mr *MockLoadBalancerClientMockRecorder) DeleteBackend(arg0, arg1 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBackend", reflect.TypeOf((*MockLoadBalancerClient)(nil).DeleteBackend), arg0, arg1) } +// DeleteBackendSet mocks base method. +func (m *MockLoadBalancerClient) DeleteBackendSet(arg0 context.Context, arg1 loadbalancer.DeleteBackendSetRequest) (loadbalancer.DeleteBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBackendSet", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.DeleteBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteBackendSet indicates an expected call of DeleteBackendSet. +func (mr *MockLoadBalancerClientMockRecorder) DeleteBackendSet(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBackendSet", reflect.TypeOf((*MockLoadBalancerClient)(nil).DeleteBackendSet), arg0, arg1) +} + +// DeleteListener mocks base method. +func (m *MockLoadBalancerClient) DeleteListener(arg0 context.Context, arg1 loadbalancer.DeleteListenerRequest) (loadbalancer.DeleteListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteListener", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.DeleteListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteListener indicates an expected call of DeleteListener. +func (mr *MockLoadBalancerClientMockRecorder) DeleteListener(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteListener", reflect.TypeOf((*MockLoadBalancerClient)(nil).DeleteListener), arg0, arg1) +} + // DeleteLoadBalancer mocks base method. func (m *MockLoadBalancerClient) DeleteLoadBalancer(arg0 context.Context, arg1 loadbalancer.DeleteLoadBalancerRequest) (loadbalancer.DeleteLoadBalancerResponse, error) { m.ctrl.T.Helper() @@ -154,3 +214,33 @@ func (mr *MockLoadBalancerClientMockRecorder) UpdateLoadBalancer(arg0, arg1 inte mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLoadBalancer", reflect.TypeOf((*MockLoadBalancerClient)(nil).UpdateLoadBalancer), arg0, arg1) } + +// UpdateBackendSet mocks base method. +func (m *MockLoadBalancerClient) UpdateBackendSet(arg0 context.Context, arg1 loadbalancer.UpdateBackendSetRequest) (loadbalancer.UpdateBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateBackendSet", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.UpdateBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateBackendSet indicates an expected call of UpdateBackendSet. +func (mr *MockLoadBalancerClientMockRecorder) UpdateBackendSet(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBackendSet", reflect.TypeOf((*MockLoadBalancerClient)(nil).UpdateBackendSet), arg0, arg1) +} + +// UpdateListener mocks base method. +func (m *MockLoadBalancerClient) UpdateListener(arg0 context.Context, arg1 loadbalancer.UpdateListenerRequest) (loadbalancer.UpdateListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateListener", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.UpdateListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateListener indicates an expected call of UpdateListener. +func (mr *MockLoadBalancerClientMockRecorder) UpdateListener(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateListener", reflect.TypeOf((*MockLoadBalancerClient)(nil).UpdateListener), arg0, arg1) +} From ed4e3ae281d97b1af1f1b08272e53c4b927213aa Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Fri, 6 Mar 2026 13:23:29 -0500 Subject: [PATCH 11/31] feat (fix) e2e test yaml for lb and nlb --- test/e2e/cluster_test.go | 2 +- .../v1beta2/cluster-template-multiple-backends-lb/cluster.yaml | 2 +- .../v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index b5395fdba..a410992eb 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -178,7 +178,7 @@ var _ = Describe("Workload cluster creation", func() { }, result) }) - It("Multiple API server backend sets on LB [DailyTests]", func() { + It("Multiple API server backend sets on LB [PRBlocking]", func() { clusterName = getClusterName(clusterNamePrefix, "multiple-backends-lb") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ ClusterProxy: bootstrapClusterProxy, diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml index 21a7aacaa..099de5385 100644 --- a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml @@ -13,7 +13,7 @@ spec: backendSetDetails: healthChecker: urlPath: /healthz - - name: apiserver-lb-backendset-secondary + - name: apiserver-lb-backendset-2 listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} backendSetDetails: healthChecker: diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml index dd439d36b..59cdf2de9 100644 --- a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml @@ -12,7 +12,7 @@ spec: backendSetDetails: healthChecker: urlPath: /healthz - - name: apiserver-lb-backendset-secondary + - name: apiserver-lb-backendset-2 listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} backendSetDetails: healthChecker: From 838d79bbdc8c4255f536bb9b0f5420e651e33d46 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Fri, 6 Mar 2026 17:26:38 -0500 Subject: [PATCH 12/31] feat (fix) LB e2e test failures --- api/v1beta1/types.go | 3 +- api/v1beta2/types.go | 3 +- cloud/scope/apiserver_lb_backendsets_test.go | 31 ++++ cloud/scope/load_balancer_reconciler.go | 128 +++++++++----- cloud/scope/load_balancer_reconciler_test.go | 163 ++++++++++++++++++ docs/src/networking/custom-networking.md | 2 + ...cluster-template-multiple-backends-lb.yaml | 154 +++++++++++++++++ 7 files changed, 437 insertions(+), 47 deletions(-) create mode 100644 templates/cluster-template-multiple-backends-lb.yaml diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index ad0eacd0a..c86588e55 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -982,7 +982,8 @@ type LoadBalancer struct { // +optional LoadBalancerId *string `json:"loadBalancerId,omitempty"` - // The NLB Spec + // Shared API server load balancer settings used for both `loadBalancerType: nlb` and `loadBalancerType: lb`. + // The field name is historical; it is not limited to the NLB implementation. // +optional NLBSpec NLBSpec `json:"nlbSpec,omitempty"` } diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index 9d9799c18..d8fca55f1 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -991,7 +991,8 @@ type LoadBalancer struct { // +optional LoadBalancerType LoadBalancerType `json:"loadBalancerType,omitempty"` - // The NLB Spec + // Shared API server load balancer settings used for both `loadBalancerType: nlb` and `loadBalancerType: lb`. + // The field name is historical; it is not limited to the NLB implementation. // +optional NLBSpec NLBSpec `json:"nlbSpec,omitempty"` } diff --git a/cloud/scope/apiserver_lb_backendsets_test.go b/cloud/scope/apiserver_lb_backendsets_test.go index 5d6115262..2f10d7d65 100644 --- a/cloud/scope/apiserver_lb_backendsets_test.go +++ b/cloud/scope/apiserver_lb_backendsets_test.go @@ -72,3 +72,34 @@ func TestBuildDesiredLBListenersAndBackendSets(t *testing.T) { t.Fatalf("expected listener to reference first backend set, got %#v", listeners[APIServerLBListener]) } } + +func TestLBSpecPreservesBackendSets(t *testing.T) { + secondaryPort := int32(9345) + scope := &ClusterScope{ + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{ + OCICluster: &infrastructurev1beta2.OCICluster{ + Spec: infrastructurev1beta2.OCIClusterSpec{ + NetworkSpec: infrastructurev1beta2.NetworkSpec{ + APIServerLB: infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "apiserver-lb-backendset-2", ListenerPort: &secondaryPort}, + }, + }, + }, + }, + }, + }, + }, + } + + got := scope.LBSpec() + if len(got.NLBSpec.BackendSets) != 2 { + t.Fatalf("expected backend sets to be preserved, got %#v", got.NLBSpec.BackendSets) + } + if got.NLBSpec.BackendSets[1].Name != "apiserver-lb-backendset-2" { + t.Fatalf("expected secondary backend set to be preserved, got %#v", got.NLBSpec.BackendSets) + } +} diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index e65b6c88e..e34e60de7 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -68,7 +68,10 @@ func (s *ClusterScope) ReconcileApiServerLB(ctx context.Context) error { Host: *lbIP, Port: s.APIServerPort(), }) - return err + if err := s.reconcileLBResources(ctx, desiredApiServerLb); err != nil { + return err + } + return nil } // DeleteApiServerLB retrieves and attempts to delete the Load Balancer if found. @@ -97,10 +100,12 @@ func (s *ClusterScope) DeleteApiServerLB(ctx context.Context) error { return nil } -// LBSpec builds the LoadBalancer from the ClusterScope and returns it +// LBSpec builds the LoadBalancer from the ClusterScope and returns it. +// Note: `NLBSpec` is the shared API server LB configuration for both OCI NLB and LBaaS paths. func (s *ClusterScope) LBSpec() infrastructurev1beta2.LoadBalancer { lbSpec := infrastructurev1beta2.LoadBalancer{ - Name: s.GetControlPlaneLoadBalancerName(), + Name: s.GetControlPlaneLoadBalancerName(), + NLBSpec: s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec, } return lbSpec } @@ -151,6 +156,47 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu } desiredListeners, desiredBackendSets := s.buildDesiredLBListenersAndBackendSets(lb) + for name, desired := range desiredBackendSets { + actual, exists := actualResp.LoadBalancer.BackendSets[name] + if !exists { + resp, err := s.LoadBalancerClient.CreateBackendSet(ctx, loadbalancer.CreateBackendSetRequest{ + LoadBalancerId: lbID, + CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ + Name: common.String(name), + Policy: desired.Policy, + HealthChecker: desired.HealthChecker, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to create lb backend set %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting create backend set %q", name) + } + continue + } + if !ptr.StringEquals(actual.Policy, ptr.ToString(desired.Policy)) || + actual.HealthChecker == nil || desired.HealthChecker == nil || + !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || + !ptr.StringEquals(actual.HealthChecker.Protocol, ptr.ToString(desired.HealthChecker.Protocol)) { + resp, err := s.LoadBalancerClient.UpdateBackendSet(ctx, loadbalancer.UpdateBackendSetRequest{ + LoadBalancerId: lbID, + BackendSetName: common.String(name), + UpdateBackendSetDetails: loadbalancer.UpdateBackendSetDetails{ + Policy: desired.Policy, + Backends: []loadbalancer.BackendDetails{}, + HealthChecker: desired.HealthChecker, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update lb backend set %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update backend set %q", name) + } + } + } + for name, desired := range desiredListeners { actual, exists := actualResp.LoadBalancer.Listeners[name] if !exists { @@ -206,47 +252,6 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu return errors.Wrapf(err, "failed awaiting delete listener %q", name) } } - - for name, desired := range desiredBackendSets { - actual, exists := actualResp.LoadBalancer.BackendSets[name] - if !exists { - resp, err := s.LoadBalancerClient.CreateBackendSet(ctx, loadbalancer.CreateBackendSetRequest{ - LoadBalancerId: lbID, - CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ - Name: common.String(name), - Policy: desired.Policy, - HealthChecker: desired.HealthChecker, - }, - }) - if err != nil { - return errors.Wrapf(err, "failed to create lb backend set %q", name) - } - if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { - return errors.Wrapf(err, "failed awaiting create backend set %q", name) - } - continue - } - if !ptr.StringEquals(actual.Policy, ptr.ToString(desired.Policy)) || - actual.HealthChecker == nil || desired.HealthChecker == nil || - !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || - !ptr.StringEquals(actual.HealthChecker.Protocol, ptr.ToString(desired.HealthChecker.Protocol)) { - resp, err := s.LoadBalancerClient.UpdateBackendSet(ctx, loadbalancer.UpdateBackendSetRequest{ - LoadBalancerId: lbID, - BackendSetName: common.String(name), - UpdateBackendSetDetails: loadbalancer.UpdateBackendSetDetails{ - Policy: desired.Policy, - Backends: []loadbalancer.BackendDetails{}, - HealthChecker: desired.HealthChecker, - }, - }) - if err != nil { - return errors.Wrapf(err, "failed to update lb backend set %q", name) - } - if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { - return errors.Wrapf(err, "failed awaiting update backend set %q", name) - } - } - } for name, actual := range actualResp.LoadBalancer.BackendSets { if _, keep := desiredBackendSets[name]; keep { continue @@ -393,11 +398,44 @@ func (s *ClusterScope) getLoadbalancerIp(lb loadbalancer.LoadBalancer) (*string, } // IsLBEqual determines if the actual loadbalancer.LoadBalancer is equal to the desired. -// Equality is determined by DisplayName, FreeformTags and DefinedTags matching. +// Equality is determined by the LB identity plus desired listener/backend-set resources matching. func (s *ClusterScope) IsLBEqual(actual *loadbalancer.LoadBalancer, desired infrastructurev1beta2.LoadBalancer) bool { if desired.Name != *actual.DisplayName { return false } + + desiredListeners, desiredBackendSets := s.buildDesiredLBListenersAndBackendSets(desired) + if len(actual.Listeners) != len(desiredListeners) { + return false + } + for name, desiredListener := range desiredListeners { + actualListener, exists := actual.Listeners[name] + if !exists { + return false + } + if !intPtrEqual(actualListener.Port, desiredListener.Port) || + !ptr.StringEquals(actualListener.DefaultBackendSetName, ptr.ToString(desiredListener.DefaultBackendSetName)) || + !ptr.StringEquals(actualListener.Protocol, ptr.ToString(desiredListener.Protocol)) { + return false + } + } + + if len(actual.BackendSets) != len(desiredBackendSets) { + return false + } + for name, desiredBackendSet := range desiredBackendSets { + actualBackendSet, exists := actual.BackendSets[name] + if !exists { + return false + } + if !ptr.StringEquals(actualBackendSet.Policy, ptr.ToString(desiredBackendSet.Policy)) || + actualBackendSet.HealthChecker == nil || desiredBackendSet.HealthChecker == nil || + !intPtrEqual(actualBackendSet.HealthChecker.Port, desiredBackendSet.HealthChecker.Port) || + !ptr.StringEquals(actualBackendSet.HealthChecker.Protocol, ptr.ToString(desiredBackendSet.HealthChecker.Protocol)) { + return false + } + } + return true } diff --git a/cloud/scope/load_balancer_reconciler_test.go b/cloud/scope/load_balancer_reconciler_test.go index 8ac36faa2..3829679ef 100644 --- a/cloud/scope/load_balancer_reconciler_test.go +++ b/cloud/scope/load_balancer_reconciler_test.go @@ -103,6 +103,24 @@ func TestLBReconciliation(t *testing.T) { DefinedTags: make(map[string]map[string]interface{}), IsPrivate: common.Bool(false), DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + }, IpAddresses: []loadbalancer.IpAddress{ { IpAddress: common.String("2.2.2.2"), @@ -316,6 +334,36 @@ func TestLBReconciliation(t *testing.T) { }, }, }, nil) + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })). + Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Id: common.String("lb-id"), + FreeformTags: tags, + IsPrivate: common.Bool(false), + DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + }, + IpAddresses: []loadbalancer.IpAddress{{IpAddress: common.String("2.2.2.2"), IsPublic: common.Bool(true)}}, + }, + }, nil) }, }, @@ -411,6 +459,24 @@ func TestLBReconciliation(t *testing.T) { }, }, }, nil) + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })). + Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Id: common.String("lb-id"), + FreeformTags: tags, + IsPrivate: common.Bool(false), + DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: {Name: common.String(APIServerLBListener), Protocol: common.String("TCP"), Port: common.Int(6443), DefaultBackendSetName: common.String(APIServerLBBackendSetName)}, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: {Name: common.String(APIServerLBBackendSetName), Policy: common.String("ROUND_ROBIN"), HealthChecker: &loadbalancer.HealthChecker{Port: common.Int(6443), Protocol: common.String("TCP")}}, + }, + IpAddresses: []loadbalancer.IpAddress{{IpAddress: common.String("2.2.2.2"), IsPublic: common.Bool(true)}}, + }, + }, nil) }, }, @@ -932,3 +998,100 @@ func TestLBDeletion(t *testing.T) { }) } } + +func TestReconcileLBResources_CreatesBackendSetsBeforeListeners(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + lbClient := mock_lb.NewMockLoadBalancerClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("lb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + LoadBalancerClient: lbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{OCICluster: ociCluster}, + Client: client, + }) + g.Expect(err).To(BeNil()) + + secondaryPort := int32(9345) + desiredLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "apiserver-lb-backendset-2", ListenerPort: &secondaryPort}, + }, + }, + } + secondaryListenerName := desiredAPIServerListenerName(1, 2, "apiserver-lb-backendset-2") + + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })).Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + }, + }, + }, nil) + + gomock.InOrder( + lbClient.EXPECT().CreateBackendSet(gomock.Any(), gomock.Eq(loadbalancer.CreateBackendSetRequest{ + LoadBalancerId: common.String("lb-id"), + CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ + Name: common.String("apiserver-lb-backendset-2"), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthCheckerDetails{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + })).Return(loadbalancer.CreateBackendSetResponse{OpcWorkRequestId: common.String("wr-backend-set")}, nil), + lbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(loadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-backend-set"), + })).Return(loadbalancer.GetWorkRequestResponse{ + WorkRequest: loadbalancer.WorkRequest{LifecycleState: loadbalancer.WorkRequestLifecycleStateSucceeded}, + }, nil), + lbClient.EXPECT().CreateListener(gomock.Any(), gomock.Eq(loadbalancer.CreateListenerRequest{ + LoadBalancerId: common.String("lb-id"), + CreateListenerDetails: loadbalancer.CreateListenerDetails{ + Name: common.String(secondaryListenerName), + DefaultBackendSetName: common.String("apiserver-lb-backendset-2"), + Port: common.Int(9345), + Protocol: common.String("TCP"), + }, + })).Return(loadbalancer.CreateListenerResponse{OpcWorkRequestId: common.String("wr-listener")}, nil), + lbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(loadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-listener"), + })).Return(loadbalancer.GetWorkRequestResponse{ + WorkRequest: loadbalancer.WorkRequest{LifecycleState: loadbalancer.WorkRequestLifecycleStateSucceeded}, + }, nil), + ) + + g.Expect(cs.reconcileLBResources(context.Background(), desiredLB)).To(Succeed()) +} diff --git a/docs/src/networking/custom-networking.md b/docs/src/networking/custom-networking.md index 0b0cf1c72..303ddbb9a 100644 --- a/docs/src/networking/custom-networking.md +++ b/docs/src/networking/custom-networking.md @@ -338,6 +338,8 @@ spec: Equivalent configuration for OCI LBaaS is supported by setting `loadBalancerType: "lb"`: +> Note: `nlbSpec` is currently the shared API-server load balancer configuration field for both OCI NLB and OCI LBaaS. The field name is historical; LB multi-backend-set configuration is also defined under `nlbSpec`. + ```yaml apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: OCICluster diff --git a/templates/cluster-template-multiple-backends-lb.yaml b/templates/cluster-template-multiple-backends-lb.yaml new file mode 100644 index 000000000..91e9d8883 --- /dev/null +++ b/templates/cluster-template-multiple-backends-lb.yaml @@ -0,0 +1,154 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: OCICluster + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + compartmentId: "${OCI_COMPARTMENT_ID}" + networkSpec: + apiServerLoadBalancer: + loadBalancerType: "lb" + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: apiserver-lb-backendset-2 + listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} + backendSetDetails: + healthChecker: + urlPath: /readyz +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + version: "${KUBERNETES_VERSION}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: OCIMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: ${KUBERNETES_VERSION} + apiServer: + certSANs: [localhost, 127.0.0.1] + dns: {} + etcd: {} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +kind: OCIMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + imageId: "${OCI_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_CONTROL_PLANE_MACHINE_TYPE=VM.Standard.E5.Flex}" + shapeConfig: + ocpus: "${OCI_CONTROL_PLANE_MACHINE_TYPE_OCPUS=1}" + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_CONTROL_PLANE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCIMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + imageId: "${OCI_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_NODE_MACHINE_TYPE=VM.Standard.E5.Flex}" + shapeConfig: + ocpus: "${OCI_NODE_MACHINE_TYPE_OCPUS=1}" + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${NODE_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: OCIMachineTemplate From a0a977c47008cc874d9970c73a88902f3c3a31db Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 11:54:58 -0400 Subject: [PATCH 13/31] feat (fix) handle multiple same port edge cases --- api/v1beta1/validator.go | 96 ++++++- api/v1beta1/validator_backendsets_test.go | 252 ++++++++++++++++++ api/v1beta2/cluster_apiserver_port.go | 54 ++++ api/v1beta2/cluster_apiserver_port_test.go | 77 ++++++ api/v1beta2/ocicluster_webhook.go | 30 ++- api/v1beta2/ocimanagedcluster_webhook.go | 30 ++- api/v1beta2/validator.go | 46 +++- api/v1beta2/validator_backendsets_test.go | 133 ++++++++- api/v1beta2/zz_generated.deepcopy.go | 30 --- cloud/ociutil/ociutil.go | 15 +- cloud/scope/defaults.go | 7 +- .../scope/network_load_balancer_reconciler.go | 28 +- 12 files changed, 714 insertions(+), 84 deletions(-) create mode 100644 api/v1beta1/validator_backendsets_test.go create mode 100644 api/v1beta2/cluster_apiserver_port.go create mode 100644 api/v1beta2/cluster_apiserver_port_test.go diff --git a/api/v1beta1/validator.go b/api/v1beta1/validator.go index a1e136f13..ddd6f00ce 100644 --- a/api/v1beta1/validator.go +++ b/api/v1beta1/validator.go @@ -23,6 +23,7 @@ import ( "fmt" "net" "regexp" + "strings" "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" "k8s.io/apimachinery/pkg/util/validation/field" @@ -32,6 +33,8 @@ const ( // can't use: \/"'[]:|<>+=;,.?*@&, Can't start with underscore. Can't end with period or hyphen. // not using . in the name to avoid issues when the name is part of DNS name. clusterNameRegex = `^[a-z0-9][a-z0-9-]{0,42}[a-z0-9]$` + + apiServerBackendSetNameRegex = `^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$` ) // invalidNameRegex is a broad regex used to validate allows names in OCI @@ -69,7 +72,7 @@ func ValidRegion(stringRegion string) bool { } // ValidateNetworkSpec validates the NetworkSpec -func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old NetworkSpec, fldPath *field.Path) field.ErrorList { +func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old NetworkSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList if len(networkSpec.Vcn.CIDR) > 0 { @@ -84,12 +87,103 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network allErrs = append(allErrs, validateNSGs(validRoles, networkSpec.Vcn.NetworkSecurityGroups, fldPath.Child("networkSecurityGroups"))...) } + allErrs = append(allErrs, validateAPIServerLBBackendSets(networkSpec.APIServerLB.NLBSpec, apiServerPort, fldPath.Child("apiServerLoadBalancer").Child("nlbSpec"))...) + if len(allErrs) == 0 { return nil } return allErrs } +func validateAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + legacyConfigured := spec.BackendSetDetails != (BackendSetDetails{}) + if len(spec.BackendSets) > 0 && legacyConfigured { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child("backendSetDetails"), + "cannot be set when backendSets is configured; move legacy backendSetDetails into backendSets", + )) + } + + nameRegex := regexp.MustCompile(apiServerBackendSetNameRegex) + seen := map[string]int{} + for i, backendSet := range spec.BackendSets { + namePath := fldPath.Child("backendSets").Index(i).Child("name") + name := strings.TrimSpace(backendSet.Name) + if name == "" { + allErrs = append(allErrs, field.Required(namePath, "must not be empty")) + continue + } + if !nameRegex.MatchString(name) { + allErrs = append(allErrs, field.Invalid( + namePath, + backendSet.Name, + "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$ (1-32 chars; alphanumeric, '-', '_')", + )) + } + if firstIdx, ok := seen[name]; ok { + allErrs = append(allErrs, field.Invalid( + namePath, + backendSet.Name, + fmt.Sprintf("duplicate backend set name, already used at backendSets[%d].name", firstIdx), + )) + continue + } + seen[name] = i + } + + usedPorts := map[int32]int{} + omittedListenerPortIndex := -1 + for i, backendSet := range spec.BackendSets { + portPath := fldPath.Child("backendSets").Index(i).Child("listenerPort") + if backendSet.ListenerPort != nil { + port := *backendSet.ListenerPort + if port < 1 || port > 65535 { + allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) + continue + } + } + + port, known := effectiveAPIServerListenerPort(backendSet.ListenerPort, apiServerPort) + if !known { + if omittedListenerPortIndex >= 0 { + allErrs = append(allErrs, field.Invalid( + portPath, + backendSet.ListenerPort, + fmt.Sprintf("multiple backend sets omit listenerPort, already omitted at backendSets[%d].listenerPort", omittedListenerPortIndex), + )) + continue + } + omittedListenerPortIndex = i + continue + } + + if firstIdx, ok := usedPorts[port]; ok { + allErrs = append(allErrs, field.Invalid( + portPath, + backendSet.ListenerPort, + fmt.Sprintf("duplicate effective listenerPort %d, already used at backendSets[%d].listenerPort", port, firstIdx), + )) + continue + } + usedPorts[port] = i + } + + return allErrs +} + +func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { + if listenerPort != nil { + return *listenerPort, true + } + if apiServerPort == nil { + return 0, false + } + + return *apiServerPort, true +} + // validateVCNCIDR validates the CIDR of a VNC. func validateVCNCIDR(vncCIDR string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta1/validator_backendsets_test.go b/api/v1beta1/validator_backendsets_test.go new file mode 100644 index 000000000..092b10811 --- /dev/null +++ b/api/v1beta1/validator_backendsets_test.go @@ -0,0 +1,252 @@ +package v1beta1 + +import ( + "strings" + "testing" + + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func TestValidateNetworkSpec_BackendSets(t *testing.T) { + t.Run("rejects mixed legacy and canonical fields", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "new-set"}}, + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for mixed fields") + } + if !strings.Contains(errs[0].Error(), "backendSetDetails") { + t.Fatalf("expected backendSetDetails path in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects duplicate backend set names", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "dup"}, {Name: "dup"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate names") + } + if !strings.Contains(errs[0].Error(), "duplicate backend set name") { + t.Fatalf("expected duplicate-name guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects invalid backend set name format", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "bad name"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for invalid name format") + } + if !strings.Contains(errs[0].Error(), "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$") { + t.Fatalf("expected regex guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts canonical backend sets with distinct effective ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "apiserver_a"}, {Name: "apiserver-b", ListenerPort: int32Ptr(9345)}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("rejects duplicate listener ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a", ListenerPort: int32Ptr(9345)}, + {Name: "set-b", ListenerPort: int32Ptr(9345)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate listener ports") + } + if !strings.Contains(errs[0].Error(), "duplicate effective listenerPort 9345") { + t.Fatalf("expected duplicate effective listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects multiple omitted listener ports even when cluster API server port is unknown", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b"}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for multiple omitted listener ports") + } + if !strings.Contains(errs[0].Error(), "multiple backend sets omit listenerPort") { + t.Fatalf("expected omitted listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts omitted and explicit listener ports when cluster API server port is unknown", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(6443)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("rejects omitted and explicit listener ports when cluster API server port is provided", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(7443)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate effective listener ports") + } + if !strings.Contains(errs[0].Error(), "duplicate effective listenerPort 7443") { + t.Fatalf("expected duplicate effective listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts explicit listener port that differs from provided cluster API server port", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(6443)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("accepts single omitted listener port", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "set-a"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) +} + +func int32Ptr(v int32) *int32 { + return &v +} diff --git a/api/v1beta2/cluster_apiserver_port.go b/api/v1beta2/cluster_apiserver_port.go new file mode 100644 index 000000000..79881bc94 --- /dev/null +++ b/api/v1beta2/cluster_apiserver_port.go @@ -0,0 +1,54 @@ +package v1beta2 + +import ( + "context" + + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + pkgerrors "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func lookupClusterAPIServerPort(ctx context.Context, c client.Client, obj metav1.ObjectMeta) (*int32, error) { + if c == nil { + return nil, nil + } + + cluster, err := util.GetOwnerCluster(ctx, c, obj) + if err != nil { + return nil, err + } + if cluster == nil { + cluster, err = util.GetClusterFromMetadata(ctx, c, obj) + if err != nil { + if pkgerrors.Cause(err) == util.ErrNoCluster { + return nil, nil + } + return nil, err + } + } + if cluster == nil { + return nil, nil + } + + port := cluster.Spec.ClusterNetwork.APIServerPort + if port == 0 { + port = ociutil.DefaultAPIServerPort + } + return &port, nil +} + +func newClusterOwnerReference(cluster *clusterv1.Cluster) metav1.OwnerReference { + controller := true + blockOwnerDeletion := true + return metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + Controller: &controller, + BlockOwnerDeletion: &blockOwnerDeletion, + } +} diff --git a/api/v1beta2/cluster_apiserver_port_test.go b/api/v1beta2/cluster_apiserver_port_test.go new file mode 100644 index 000000000..88373239c --- /dev/null +++ b/api/v1beta2/cluster_apiserver_port_test.go @@ -0,0 +1,77 @@ +package v1beta2 + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + ctrlclientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestLookupClusterAPIServerPort(t *testing.T) { + t.Run("uses owner cluster api server port when set", func(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add cluster api scheme: %v", err) + } + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + UID: "cluster-uid", + }, + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: clusterv1.ClusterNetwork{APIServerPort: 7443}, + }, + } + + client := ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster).Build() + objMeta := metav1.ObjectMeta{ + Name: "test-oci-cluster", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{newClusterOwnerReference(cluster)}, + } + + port, err := lookupClusterAPIServerPort(context.Background(), client, objMeta) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if port == nil || *port != 7443 { + t.Fatalf("expected api server port 7443, got %v", port) + } + }) + + t.Run("falls back to cluster-name label and default port", func(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add cluster api scheme: %v", err) + } + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + } + + client := ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster).Build() + objMeta := metav1.ObjectMeta{ + Name: "test-oci-cluster", + Namespace: "default", + Labels: map[string]string{ + clusterv1.ClusterNameLabel: cluster.Name, + }, + } + + port, err := lookupClusterAPIServerPort(context.Background(), client, objMeta) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if port == nil || *port != 6443 { + t.Fatalf("expected api server port 6443, got %v", port) + } + }) +} diff --git a/api/v1beta2/ocicluster_webhook.go b/api/v1beta2/ocicluster_webhook.go index 17c614281..412438130 100644 --- a/api/v1beta2/ocicluster_webhook.go +++ b/api/v1beta2/ocicluster_webhook.go @@ -29,13 +29,17 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) var clusterlogger = ctrl.Log.WithName("ocicluster-resource") -type OCIClusterWebhook struct{} +// +kubebuilder:object:generate=false +type OCIClusterWebhook struct { + Client client.Client +} var ( _ webhook.CustomDefaulter = &OCIClusterWebhook{} @@ -63,7 +67,7 @@ func (*OCIClusterWebhook) Default(_ context.Context, obj runtime.Object) error { } func (c *OCICluster) SetupWebhookWithManager(mgr ctrl.Manager) error { - w := new(OCIClusterWebhook) + w := &OCIClusterWebhook{Client: mgr.GetClient()} return ctrl.NewWebhookManagedBy(mgr). For(c). WithDefaulter(w). @@ -72,7 +76,7 @@ func (c *OCICluster) SetupWebhookWithManager(mgr ctrl.Manager) error { } // ValidateCreate implements webhook.Validator so a webhook will be registered for the type. -func (*OCIClusterWebhook) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { +func (w *OCIClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { c, ok := obj.(*OCICluster) if !ok { return nil, fmt.Errorf("expected an OCICluster object but got %T", c) @@ -80,6 +84,11 @@ func (*OCIClusterWebhook) ValidateCreate(_ context.Context, obj runtime.Object) clusterlogger.Info("validate update cluster", "name", c.Name) + apiServerPort, err := lookupClusterAPIServerPort(ctx, w.Client, c.ObjectMeta) + if err != nil { + return nil, err + } + var allErrs field.ErrorList var ipv6hextets []*string var hextatassigned bool @@ -163,7 +172,7 @@ func (*OCIClusterWebhook) ValidateCreate(_ context.Context, obj runtime.Object) } } - allErrs = append(allErrs, c.validate(nil)...) + allErrs = append(allErrs, c.validate(nil, apiServerPort)...) if len(allErrs) == 0 { return nil, nil @@ -184,7 +193,7 @@ func (*OCIClusterWebhook) ValidateDelete(_ context.Context, obj runtime.Object) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. -func (*OCIClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runtime.Object) (admission.Warnings, error) { +func (w *OCIClusterWebhook) ValidateUpdate(ctx context.Context, oldRaw, newObj runtime.Object) (admission.Warnings, error) { c, ok := newObj.(*OCICluster) if !ok { return nil, fmt.Errorf("expected an OCICluster object but got %T", c) @@ -192,6 +201,11 @@ func (*OCIClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runti clusterlogger.Info("validate update cluster", "name", c.Name) + apiServerPort, err := lookupClusterAPIServerPort(ctx, w.Client, c.ObjectMeta) + if err != nil { + return nil, err + } + var allErrs field.ErrorList oldCluster, ok := oldRaw.(*OCICluster) @@ -262,7 +276,7 @@ func (*OCIClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runti } } - allErrs = append(allErrs, c.validate(oldCluster)...) + allErrs = append(allErrs, c.validate(oldCluster, apiServerPort)...) if len(allErrs) == 0 { return nil, nil @@ -271,7 +285,7 @@ func (*OCIClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runti return nil, apierrors.NewInvalid(c.GroupVersionKind().GroupKind(), c.Name, allErrs) } -func (c *OCICluster) validate(old *OCICluster) field.ErrorList { +func (c *OCICluster) validate(old *OCICluster, apiServerPort *int32) field.ErrorList { var allErrs field.ErrorList var oldNetworkSpec NetworkSpec @@ -279,7 +293,7 @@ func (c *OCICluster) validate(old *OCICluster) field.ErrorList { oldNetworkSpec = old.Spec.NetworkSpec } - allErrs = append(allErrs, ValidateNetworkSpec(OCIClusterSubnetRoles, c.Spec.NetworkSpec, oldNetworkSpec, field.NewPath("spec").Child("networkSpec"))...) + allErrs = append(allErrs, ValidateNetworkSpec(OCIClusterSubnetRoles, c.Spec.NetworkSpec, oldNetworkSpec, apiServerPort, field.NewPath("spec").Child("networkSpec"))...) allErrs = append(allErrs, ValidateClusterName(c.Name)...) if len(c.Spec.CompartmentId) <= 0 { diff --git a/api/v1beta2/ocimanagedcluster_webhook.go b/api/v1beta2/ocimanagedcluster_webhook.go index 5a4e67771..38dc829db 100644 --- a/api/v1beta2/ocimanagedcluster_webhook.go +++ b/api/v1beta2/ocimanagedcluster_webhook.go @@ -27,13 +27,17 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) var managedclusterlogger = ctrl.Log.WithName("ocimanagedcluster-resource") -type OCIManagedClusterWebhook struct{} +// +kubebuilder:object:generate=false +type OCIManagedClusterWebhook struct { + Client client.Client +} var ( _ webhook.CustomDefaulter = &OCIManagedClusterWebhook{} @@ -119,7 +123,7 @@ func (*OCIManagedClusterWebhook) Default(_ context.Context, obj runtime.Object) } func (c *OCIManagedCluster) SetupWebhookWithManager(mgr ctrl.Manager) error { - w := new(OCIManagedClusterWebhook) + w := &OCIManagedClusterWebhook{Client: mgr.GetClient()} return ctrl.NewWebhookManagedBy(mgr). For(c). WithDefaulter(w). @@ -128,16 +132,21 @@ func (c *OCIManagedCluster) SetupWebhookWithManager(mgr ctrl.Manager) error { } // ValidateCreate implements webhook.Validator so a webhook will be registered for the type. -func (*OCIManagedClusterWebhook) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { +func (w *OCIManagedClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { c, ok := obj.(*OCIManagedCluster) if !ok { return nil, fmt.Errorf("expected an OCIManagedCluster object but got %T", c) } managedclusterlogger.Info("validate create cluster", "name", c.Name) + apiServerPort, err := lookupClusterAPIServerPort(ctx, w.Client, c.ObjectMeta) + if err != nil { + return nil, err + } + var allErrs field.ErrorList - allErrs = append(allErrs, c.validate(nil)...) + allErrs = append(allErrs, c.validate(nil, apiServerPort)...) if len(allErrs) == 0 { return nil, nil @@ -159,7 +168,7 @@ func (*OCIManagedClusterWebhook) ValidateDelete(_ context.Context, obj runtime.O } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. -func (*OCIManagedClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runtime.Object) (admission.Warnings, error) { +func (w *OCIManagedClusterWebhook) ValidateUpdate(ctx context.Context, oldRaw, newObj runtime.Object) (admission.Warnings, error) { c, ok := newObj.(*OCIManagedCluster) if !ok { return nil, fmt.Errorf("expected an OCIManagedCluster object but got %T", c) @@ -186,7 +195,12 @@ func (*OCIManagedClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newOb allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "compartmentId"), c.Spec.CompartmentId, "field is immutable")) } - allErrs = append(allErrs, c.validate(oldCluster)...) + apiServerPort, err := lookupClusterAPIServerPort(ctx, w.Client, c.ObjectMeta) + if err != nil { + return nil, err + } + + allErrs = append(allErrs, c.validate(oldCluster, apiServerPort)...) if len(allErrs) == 0 { return nil, nil @@ -195,7 +209,7 @@ func (*OCIManagedClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newOb return nil, apierrors.NewInvalid(c.GroupVersionKind().GroupKind(), c.Name, allErrs) } -func (c *OCIManagedCluster) validate(old *OCIManagedCluster) field.ErrorList { +func (c *OCIManagedCluster) validate(old *OCIManagedCluster, apiServerPort *int32) field.ErrorList { var allErrs field.ErrorList var oldNetworkSpec NetworkSpec @@ -203,7 +217,7 @@ func (c *OCIManagedCluster) validate(old *OCIManagedCluster) field.ErrorList { oldNetworkSpec = old.Spec.NetworkSpec } - allErrs = append(allErrs, ValidateNetworkSpec(OCIManagedClusterSubnetRoles, c.Spec.NetworkSpec, oldNetworkSpec, field.NewPath("spec").Child("networkSpec"))...) + allErrs = append(allErrs, ValidateNetworkSpec(OCIManagedClusterSubnetRoles, c.Spec.NetworkSpec, oldNetworkSpec, apiServerPort, field.NewPath("spec").Child("networkSpec"))...) allErrs = append(allErrs, ValidateClusterName(c.Name)...) if len(c.Spec.CompartmentId) <= 0 { diff --git a/api/v1beta2/validator.go b/api/v1beta2/validator.go index 9f67cc442..b22a40b25 100644 --- a/api/v1beta2/validator.go +++ b/api/v1beta2/validator.go @@ -111,7 +111,7 @@ func ValidRegion(stringRegion string) bool { } // ValidateNetworkSpec validates the NetworkSpec -func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old NetworkSpec, fldPath *field.Path) field.ErrorList { +func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old NetworkSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList if len(networkSpec.Vcn.CIDR) > 0 { @@ -128,7 +128,7 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network nsgErrors := validateNSGs(validRoles, networkSpec.Vcn.NetworkSecurityGroup.List, fldPath.Child("networkSecurityGroups")) allErrs = append(allErrs, nsgErrors...) } - allErrs = append(allErrs, validateAPIServerLBBackendSets(networkSpec.APIServerLB.NLBSpec, fldPath.Child("apiServerLoadBalancer").Child("nlbSpec"))...) + allErrs = append(allErrs, validateAPIServerLBBackendSets(networkSpec.APIServerLB.NLBSpec, apiServerPort, fldPath.Child("apiServerLoadBalancer").Child("nlbSpec"))...) if len(allErrs) == 0 { return nil @@ -136,7 +136,7 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network return allErrs } -func validateAPIServerLBBackendSets(spec NLBSpec, fldPath *field.Path) field.ErrorList { +func validateAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList legacyConfigured := spec.BackendSetDetails != (BackendSetDetails{}) @@ -175,21 +175,36 @@ func validateAPIServerLBBackendSets(spec NLBSpec, fldPath *field.Path) field.Err } usedPorts := map[int32]int{} + omittedListenerPortIndex := -1 for i, backendSet := range spec.BackendSets { portPath := fldPath.Child("backendSets").Index(i).Child("listenerPort") - if backendSet.ListenerPort == nil { - continue + if backendSet.ListenerPort != nil { + port := *backendSet.ListenerPort + if port < 1 || port > 65535 { + allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) + continue + } } - port := *backendSet.ListenerPort - if port < 1 || port > 65535 { - allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) + + port, known := effectiveAPIServerListenerPort(backendSet.ListenerPort, apiServerPort) + if !known { + if omittedListenerPortIndex >= 0 { + allErrs = append(allErrs, field.Invalid( + portPath, + backendSet.ListenerPort, + fmt.Sprintf("multiple backend sets omit listenerPort, already omitted at backendSets[%d].listenerPort", omittedListenerPortIndex), + )) + continue + } + omittedListenerPortIndex = i continue } + if firstIdx, ok := usedPorts[port]; ok { allErrs = append(allErrs, field.Invalid( portPath, - port, - fmt.Sprintf("duplicate listenerPort, already used at backendSets[%d].listenerPort", firstIdx), + backendSet.ListenerPort, + fmt.Sprintf("duplicate effective listenerPort %d, already used at backendSets[%d].listenerPort", port, firstIdx), )) continue } @@ -199,6 +214,17 @@ func validateAPIServerLBBackendSets(spec NLBSpec, fldPath *field.Path) field.Err return allErrs } +func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { + if listenerPort != nil { + return *listenerPort, true + } + if apiServerPort == nil { + return 0, false + } + + return *apiServerPort, true +} + // validateVCNCIDR validates the CIDR of a VNC. func validateVCNCIDR(vncCIDR string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta2/validator_backendsets_test.go b/api/v1beta2/validator_backendsets_test.go index d7b15115e..3693c0071 100644 --- a/api/v1beta2/validator_backendsets_test.go +++ b/api/v1beta2/validator_backendsets_test.go @@ -22,6 +22,7 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { }, }, NetworkSpec{}, + nil, field.NewPath("spec").Child("networkSpec"), ) @@ -44,6 +45,7 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { }, }, NetworkSpec{}, + nil, field.NewPath("spec").Child("networkSpec"), ) @@ -66,6 +68,7 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { }, }, NetworkSpec{}, + nil, field.NewPath("spec").Child("networkSpec"), ) @@ -77,17 +80,18 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { } }) - t.Run("accepts canonical backend sets", func(t *testing.T) { + t.Run("accepts canonical backend sets with distinct effective ports", func(t *testing.T) { errs := ValidateNetworkSpec( OCIClusterSubnetRoles, NetworkSpec{ APIServerLB: LoadBalancer{ NLBSpec: NLBSpec{ - BackendSets: []NLBBackendSet{{Name: "apiserver_a"}, {Name: "apiserver-b"}}, + BackendSets: []NLBBackendSet{{Name: "apiserver_a"}, {Name: "apiserver-b", ListenerPort: int32Ptr(9345)}}, }, }, }, NetworkSpec{}, + nil, field.NewPath("spec").Child("networkSpec"), ) @@ -110,14 +114,135 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { }, }, NetworkSpec{}, + nil, field.NewPath("spec").Child("networkSpec"), ) if len(errs) == 0 { t.Fatalf("expected validation error for duplicate listener ports") } - if !strings.Contains(errs[0].Error(), "duplicate listenerPort") { - t.Fatalf("expected duplicate listener port guidance in error, got %q", errs[0].Error()) + if !strings.Contains(errs[0].Error(), "duplicate effective listenerPort 9345") { + t.Fatalf("expected duplicate effective listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects multiple omitted listener ports even when cluster API server port is unknown", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b"}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for multiple omitted listener ports") + } + if !strings.Contains(errs[0].Error(), "multiple backend sets omit listenerPort") { + t.Fatalf("expected omitted listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts omitted and explicit listener ports when cluster API server port is unknown", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(6443)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("rejects omitted and explicit listener ports when cluster API server port is provided", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(7443)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate effective listener ports") + } + if !strings.Contains(errs[0].Error(), "duplicate effective listenerPort 7443") { + t.Fatalf("expected duplicate effective listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts explicit listener port that differs from provided cluster API server port", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(6443)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("accepts single omitted listener port", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "set-a"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) } }) } diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index b3084eddf..a91fa83d7 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -1892,21 +1892,6 @@ func (in *OCIClusterTemplateSpec) DeepCopy() *OCIClusterTemplateSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OCIClusterWebhook) DeepCopyInto(out *OCIClusterWebhook) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCIClusterWebhook. -func (in *OCIClusterWebhook) DeepCopy() *OCIClusterWebhook { - if in == nil { - return nil - } - out := new(OCIClusterWebhook) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OCIMachine) DeepCopyInto(out *OCIMachine) { *out = *in @@ -2480,21 +2465,6 @@ func (in *OCIManagedClusterTemplateSpec) DeepCopy() *OCIManagedClusterTemplateSp return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OCIManagedClusterWebhook) DeepCopyInto(out *OCIManagedClusterWebhook) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCIManagedClusterWebhook. -func (in *OCIManagedClusterWebhook) DeepCopy() *OCIManagedClusterWebhook { - if in == nil { - return nil - } - out := new(OCIManagedClusterWebhook) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OCIManagedControlPlane) DeepCopyInto(out *OCIManagedControlPlane) { *out = *in diff --git a/cloud/ociutil/ociutil.go b/cloud/ociutil/ociutil.go index e49fb265d..05b85b5c6 100644 --- a/cloud/ociutil/ociutil.go +++ b/cloud/ociutil/ociutil.go @@ -36,13 +36,14 @@ import ( ) const ( - WorkRequestPollInterval = 5 * time.Second - WorkRequestTimeout = 2 * time.Minute - MaxOPCRetryTokenBytes = 64 - CreatedBy = "CreatedBy" - OCIClusterAPIProvider = "OCIClusterAPIProvider" - ClusterResourceIdentifier = "ClusterResourceIdentifier" - OutOfHostCapacityErr = "Out of host capacity" + DefaultAPIServerPort int32 = 6443 + WorkRequestPollInterval = 5 * time.Second + WorkRequestTimeout = 2 * time.Minute + MaxOPCRetryTokenBytes = 64 + CreatedBy = "CreatedBy" + OCIClusterAPIProvider = "OCIClusterAPIProvider" + ClusterResourceIdentifier = "ClusterResourceIdentifier" + OutOfHostCapacityErr = "Out of host capacity" ) // ErrNotFound is for simulation during testing, OCI SDK does not have a way diff --git a/cloud/scope/defaults.go b/cloud/scope/defaults.go index 77c97e3b9..c062aae6d 100644 --- a/cloud/scope/defaults.go +++ b/cloud/scope/defaults.go @@ -16,7 +16,10 @@ package scope -import "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" +import ( + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" +) const ( VcnDefaultCidr = "10.0.0.0/16" @@ -24,7 +27,7 @@ const ( ControlPlaneMachineSubnetDefaultCIDR = "10.0.0.0/29" WorkerSubnetDefaultCIDR = "10.0.64.0/20" ServiceLoadBalancerDefaultCIDR = "10.0.0.32/27" - ApiServerPort = 6443 + ApiServerPort = ociutil.DefaultAPIServerPort APIServerLBBackendSetName = "apiserver-lb-backendset" APIServerLBListener = "apiserver-lb-listener" SGWServiceSuffix = "-services-in-oracle-services-network" diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index 50854eb11..f66b3826a 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -227,11 +227,11 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc resp, err := extendedClient.CreateBackendSet(ctx, networkloadbalancer.CreateBackendSetRequest{ NetworkLoadBalancerId: nlbID, CreateBackendSetDetails: networkloadbalancer.CreateBackendSetDetails{ - Name: common.String(name), - Policy: desired.Policy, - HealthChecker: nlbHealthCheckerToDetails(desired.HealthChecker), - IsPreserveSource: desired.IsPreserveSource, - IsFailOpen: desired.IsFailOpen, + Name: common.String(name), + Policy: desired.Policy, + HealthChecker: nlbHealthCheckerToDetails(desired.HealthChecker), + IsPreserveSource: desired.IsPreserveSource, + IsFailOpen: desired.IsFailOpen, IsInstantFailoverEnabled: desired.IsInstantFailoverEnabled, }, }) @@ -457,16 +457,16 @@ func nlbHealthCheckerToDetails(in *networkloadbalancer.HealthChecker) *networklo return nil } return &networkloadbalancer.HealthCheckerDetails{ - Protocol: in.Protocol, - Port: in.Port, - Retries: in.Retries, - TimeoutInMillis: in.TimeoutInMillis, - IntervalInMillis: in.IntervalInMillis, - UrlPath: in.UrlPath, + Protocol: in.Protocol, + Port: in.Port, + Retries: in.Retries, + TimeoutInMillis: in.TimeoutInMillis, + IntervalInMillis: in.IntervalInMillis, + UrlPath: in.UrlPath, ResponseBodyRegex: in.ResponseBodyRegex, - ReturnCode: in.ReturnCode, - RequestData: in.RequestData, - ResponseData: in.ResponseData, + ReturnCode: in.ReturnCode, + RequestData: in.RequestData, + ResponseData: in.ResponseData, } } From 3441aa693999cd346929fed6a81a9819509c8246 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 18:31:27 -0400 Subject: [PATCH 14/31] Fix API server backend port registration --- cloud/scope/machine.go | 9 +-------- cloud/scope/machine_backendsets_test.go | 4 ++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index 46ae67b47..be732d468 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -863,14 +863,7 @@ func (m *MachineScope) desiredAPIServerBackendSetNames() []string { } func (m *MachineScope) desiredAPIServerBackendPort(backendSetName string) int32 { - defaultPort := m.OCIClusterAccessor.GetControlPlaneEndpoint().Port - backendSets := m.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec.CanonicalBackendSets() - for _, backendSet := range backendSets { - if backendSet.Name == backendSetName { - return desiredAPIServerListenerPort(defaultPort, backendSet) - } - } - return defaultPort + return m.OCIClusterAccessor.GetControlPlaneEndpoint().Port } func (m *MachineScope) createBackendRetryToken(backendSetName string, backendPort int32) *string { diff --git a/cloud/scope/machine_backendsets_test.go b/cloud/scope/machine_backendsets_test.go index 34b935c43..ac8986d96 100644 --- a/cloud/scope/machine_backendsets_test.go +++ b/cloud/scope/machine_backendsets_test.go @@ -277,10 +277,10 @@ func TestNLBCreateMembership_DefaultAndSecondaryBackendSetNaming(t *testing.T) { BackendSetName: common.String("apiserver-lb-backendset-2"), CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ IpAddress: common.String("1.1.1.1"), - Port: common.Int(9345), + Port: common.Int(6443), Name: common.String(fmt.Sprintf("test-%s", stableShortHash("apiserver-lb-backendset-2"))), }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("apiserver-lb-backendset-2:9345")), + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("apiserver-lb-backendset-2:6443")), })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-secondary")}, nil) nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ WorkRequestId: common.String("wrid-secondary"), From 8a5b56025ac0490f6c53dde1a00889ab13985de2 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 18:35:09 -0400 Subject: [PATCH 15/31] Trigger NLB reconcile on spec changes --- cloud/scope/apiserver_lb_backendsets_test.go | 64 ++++++++++++++ .../scope/network_load_balancer_reconciler.go | 41 ++++++++- .../network_load_balancer_reconciler_test.go | 87 +++++++++++++++++++ 3 files changed, 191 insertions(+), 1 deletion(-) diff --git a/cloud/scope/apiserver_lb_backendsets_test.go b/cloud/scope/apiserver_lb_backendsets_test.go index 2f10d7d65..76ce5842a 100644 --- a/cloud/scope/apiserver_lb_backendsets_test.go +++ b/cloud/scope/apiserver_lb_backendsets_test.go @@ -5,6 +5,7 @@ import ( infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" ) @@ -103,3 +104,66 @@ func TestLBSpecPreservesBackendSets(t *testing.T) { t.Fatalf("expected secondary backend set to be preserved, got %#v", got.NLBSpec.BackendSets) } } + +func TestIsNLBEqual_DetectsSpecOnlyResourceChanges(t *testing.T) { + secondaryPort := int32(9345) + scope := &ClusterScope{Cluster: &clusterv1.Cluster{}} + desired := infrastructurev1beta2.LoadBalancer{ + Name: "cluster-apiserver", + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "rollout-set", ListenerPort: &secondaryPort}, + }, + }, + } + + actual := &networkloadbalancer.NetworkLoadBalancer{ + DisplayName: common.String("cluster-apiserver"), + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + desiredAPIServerListenerName(1, 2, "rollout-set"): { + Name: common.String(desiredAPIServerListenerName(1, 2, "rollout-set")), + DefaultBackendSetName: common.String("rollout-set"), + Port: common.Int(9345), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + "rollout-set": { + Name: common.String("rollout-set"), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + }, + } + if !scope.IsNLBEqual(actual, desired) { + t.Fatalf("expected matching NLB resources to be equal") + } + + delete(actual.Listeners, desiredAPIServerListenerName(1, 2, "rollout-set")) + delete(actual.BackendSets, "rollout-set") + if scope.IsNLBEqual(actual, desired) { + t.Fatalf("expected spec-only listener/backend-set changes to make NLB unequal") + } +} diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index f66b3826a..5ff53983a 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -491,11 +491,50 @@ func (s *ClusterScope) getNetworkLoadbalancerIp(nlb networkloadbalancer.NetworkL } // IsNLBEqual determines if the actual networkloadbalancer.NetworkLoadBalancer is equal to the desired. -// Equality is determined by DisplayName +// Equality is determined by the NLB identity plus desired listener/backend-set resources matching. func (s *ClusterScope) IsNLBEqual(actual *networkloadbalancer.NetworkLoadBalancer, desired infrastructurev1beta2.LoadBalancer) bool { if desired.Name != *actual.DisplayName { return false } + + desiredListeners, desiredBackendSets := s.buildDesiredNLBListenersAndBackendSets(desired) + if len(actual.Listeners) != len(desiredListeners) { + return false + } + for name, desiredListener := range desiredListeners { + actualListener, exists := actual.Listeners[name] + if !exists { + return false + } + if !intPtrEqual(actualListener.Port, desiredListener.Port) || + !ptr.StringEquals(actualListener.DefaultBackendSetName, ptr.ToString(desiredListener.DefaultBackendSetName)) || + actualListener.Protocol != desiredListener.Protocol { + return false + } + } + + if len(actual.BackendSets) != len(desiredBackendSets) { + return false + } + for name, desiredBackendSet := range desiredBackendSets { + actualBackendSet, exists := actual.BackendSets[name] + if !exists { + return false + } + if actualBackendSet.HealthChecker == nil || desiredBackendSet.HealthChecker == nil { + return false + } + if actualBackendSet.Policy != desiredBackendSet.Policy || + !boolPtrEqual(actualBackendSet.IsPreserveSource, desiredBackendSet.IsPreserveSource) || + !boolPtrEqual(actualBackendSet.IsFailOpen, desiredBackendSet.IsFailOpen) || + !boolPtrEqual(actualBackendSet.IsInstantFailoverEnabled, desiredBackendSet.IsInstantFailoverEnabled) || + !intPtrEqual(actualBackendSet.HealthChecker.Port, desiredBackendSet.HealthChecker.Port) || + actualBackendSet.HealthChecker.Protocol != desiredBackendSet.HealthChecker.Protocol || + !ptr.StringEquals(actualBackendSet.HealthChecker.UrlPath, ptr.ToString(desiredBackendSet.HealthChecker.UrlPath)) { + return false + } + } + return true } diff --git a/cloud/scope/network_load_balancer_reconciler_test.go b/cloud/scope/network_load_balancer_reconciler_test.go index 049a7b315..ca962e25f 100644 --- a/cloud/scope/network_load_balancer_reconciler_test.go +++ b/cloud/scope/network_load_balancer_reconciler_test.go @@ -109,8 +109,95 @@ func TestNLBReconciliation(t *testing.T) { IsPublic: common.Bool(true), }, }, + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + }, + }, + }, nil) + }, + }, + { + name: "nlb spec-only resource change triggers update", + errorExpected: false, + testSpecificSetup: func(clusterScope *ClusterScope, nlbClient *mock_nlb.MockNetworkLoadBalancerClient) { + secondaryPort := int32(9345) + clusterScope.OCIClusterAccessor.GetNetworkSpec().APIServerLB.LoadBalancerId = common.String("nlb-id") + clusterScope.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec.BackendSets = []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "rollout-set", ListenerPort: &secondaryPort}, + } + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + })). + Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + LifecycleState: networkloadbalancer.LifecycleStateActive, + Id: common.String("nlb-id"), + FreeformTags: tags, + DefinedTags: make(map[string]map[string]interface{}), + IsPrivate: common.Bool(false), + DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + IpAddresses: []networkloadbalancer.IpAddress{ + { + IpAddress: common.String("2.2.2.2"), + IsPublic: common.Bool(true), + }, + }, + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + }, }, }, nil) + nlbClient.EXPECT().UpdateNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.UpdateNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + UpdateNetworkLoadBalancerDetails: networkloadbalancer.UpdateNetworkLoadBalancerDetails{ + DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + }, + })). + Return(networkloadbalancer.UpdateNetworkLoadBalancerResponse{ + OpcWorkRequestId: common.String("wr-nlb-update"), + }, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-nlb-update"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{ + Status: networkloadbalancer.OperationStatusSucceeded, + }, + }, nil) }, }, { From 15a135a34272eadb98a6d277161cea456d92da57 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 18:39:00 -0400 Subject: [PATCH 16/31] Constrain API server listener ports --- api/v1beta1/types.go | 3 +- api/v1beta1/validator.go | 45 +++++++++++++- api/v1beta1/validator_backendsets_test.go | 52 +++++++++++++++++ api/v1beta2/types.go | 3 +- api/v1beta2/validator.go | 71 ++++++++++++++++++----- api/v1beta2/validator_backendsets_test.go | 52 +++++++++++++++++ 6 files changed, 209 insertions(+), 17 deletions(-) diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index c86588e55..2f3234da0 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -1015,7 +1015,8 @@ type NLBBackendSet struct { Name string `json:"name"` // ListenerPort is the load balancer listener port routed to this backend set. - // When omitted, the cluster API server port is used. + // When omitted, the cluster API server port is used. Explicit values are restricted + // to the supported API server listener ports. // +optional ListenerPort *int32 `json:"listenerPort,omitempty"` diff --git a/api/v1beta1/validator.go b/api/v1beta1/validator.go index ddd6f00ce..2806e5ebd 100644 --- a/api/v1beta1/validator.go +++ b/api/v1beta1/validator.go @@ -34,7 +34,8 @@ const ( // not using . in the name to avoid issues when the name is part of DNS name. clusterNameRegex = `^[a-z0-9][a-z0-9-]{0,42}[a-z0-9]$` - apiServerBackendSetNameRegex = `^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$` + apiServerBackendSetNameRegex = `^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$` + supportedSecondaryListenerPort int32 = 9345 ) // invalidNameRegex is a broad regex used to validate allows names in OCI @@ -143,6 +144,14 @@ func validateAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) continue } + if !isSupportedAPIServerListenerPort(port, apiServerPort) { + allErrs = append(allErrs, field.Invalid( + portPath, + port, + fmt.Sprintf("must be one of %s to avoid exposing arbitrary control-plane ports", supportedAPIServerListenerPortsString(apiServerPort)), + )) + continue + } } port, known := effectiveAPIServerListenerPort(backendSet.ListenerPort, apiServerPort) @@ -184,6 +193,40 @@ func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) ( return *apiServerPort, true } +func isSupportedAPIServerListenerPort(port int32, apiServerPort *int32) bool { + for _, allowedPort := range supportedAPIServerListenerPorts(apiServerPort) { + if port == allowedPort { + return true + } + } + return false +} + +func supportedAPIServerListenerPorts(apiServerPort *int32) []int32 { + ports := []int32{ociutil.DefaultAPIServerPort} + if apiServerPort != nil && *apiServerPort != ociutil.DefaultAPIServerPort { + ports = append(ports, *apiServerPort) + } + if supportedSecondaryListenerPort != ociutil.DefaultAPIServerPort { + ports = append(ports, supportedSecondaryListenerPort) + } + return ports +} + +func supportedAPIServerListenerPortsString(apiServerPort *int32) string { + values := supportedAPIServerListenerPorts(apiServerPort) + parts := make([]string, 0, len(values)) + seen := map[int32]struct{}{} + for _, value := range values { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + parts = append(parts, fmt.Sprintf("%d", value)) + } + return strings.Join(parts, ", ") +} + // validateVCNCIDR validates the CIDR of a VNC. func validateVCNCIDR(vncCIDR string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta1/validator_backendsets_test.go b/api/v1beta1/validator_backendsets_test.go index 092b10811..1b6fea0ad 100644 --- a/api/v1beta1/validator_backendsets_test.go +++ b/api/v1beta1/validator_backendsets_test.go @@ -126,6 +126,31 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { } }) + t.Run("rejects unsupported listener ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a", ListenerPort: int32Ptr(10250)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for unsupported listener port") + } + if !strings.Contains(errs[0].Error(), "must be one of 6443, 9345") { + t.Fatalf("expected supported-port guidance in error, got %q", errs[0].Error()) + } + }) + t.Run("rejects multiple omitted listener ports even when cluster API server port is unknown", func(t *testing.T) { errs := ValidateNetworkSpec( OCIClusterSubnetRoles, @@ -226,6 +251,33 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { } }) + t.Run("rejects unsupported listener port when cluster API server port is provided", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(10250)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for unsupported listener port") + } + if !strings.Contains(errs[0].Error(), "must be one of 6443, 7443, 9345") { + t.Fatalf("expected supported-port guidance in error, got %q", errs[0].Error()) + } + }) + t.Run("accepts single omitted listener port", func(t *testing.T) { errs := ValidateNetworkSpec( OCIClusterSubnetRoles, diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index d8fca55f1..acc049907 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -1024,7 +1024,8 @@ type NLBBackendSet struct { Name string `json:"name"` // ListenerPort is the load balancer listener port routed to this backend set. - // When omitted, the cluster API server port is used. + // When omitted, the cluster API server port is used. Explicit values are restricted + // to the supported API server listener ports. // +optional ListenerPort *int32 `json:"listenerPort,omitempty"` diff --git a/api/v1beta2/validator.go b/api/v1beta2/validator.go index b22a40b25..059f63254 100644 --- a/api/v1beta2/validator.go +++ b/api/v1beta2/validator.go @@ -39,20 +39,21 @@ const ( ingressRulesType = "ingressRules" // Error message formats for NSG security rule validation - udpDestinationPortRangeMaxRequiredFormat = "invalid %s: UdpOptions DestinationPortRange Max may not be empty" - udpDestinationPortRangeMinRequiredFormat = "invalid %s: UdpOptions DestinationPortRange Min may not be empty" - udpSourcePortRangeMaxRequiredFormat = "invalid %s: UdpOptions SourcePortRange Max may not be empty" - udpSourcePortRangeMinRequiredFormat = "invalid %s: UdpOptions SourcePortRange Min may not be empty" - tcpDestinationPortRangeMaxRequiredFormat = "invalid %s: TcpOptions DestinationPortRange Max may not be empty" - tcpDestinationPortRangeMinRequiredFormat = "invalid %s: TcpOptions DestinationPortRange Min may not be empty" - tcpSourcePortRangeMaxRequiredFormat = "invalid %s: TcpOptions SourcePortRange Max may not be empty" - tcpSourcePortRangeMinRequiredFormat = "invalid %s: TcpOptions SourcePortRange Min may not be empty" - icmpTypeRequiredFormat = "invalid %s: IcmpOptions Type may not be empty" - destinationRequiredFormat = "invalid %s: Destination may not be empty" - sourceRequiredFormat = "invalid %s: Source may not be empty" - protocolRequiredFormat = "invalid %s: Protocol may not be empty" - invalidCIDRFormatFormat = "invalid %s: CIDR format" - apiServerBackendSetNameRegex = `^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$` + udpDestinationPortRangeMaxRequiredFormat = "invalid %s: UdpOptions DestinationPortRange Max may not be empty" + udpDestinationPortRangeMinRequiredFormat = "invalid %s: UdpOptions DestinationPortRange Min may not be empty" + udpSourcePortRangeMaxRequiredFormat = "invalid %s: UdpOptions SourcePortRange Max may not be empty" + udpSourcePortRangeMinRequiredFormat = "invalid %s: UdpOptions SourcePortRange Min may not be empty" + tcpDestinationPortRangeMaxRequiredFormat = "invalid %s: TcpOptions DestinationPortRange Max may not be empty" + tcpDestinationPortRangeMinRequiredFormat = "invalid %s: TcpOptions DestinationPortRange Min may not be empty" + tcpSourcePortRangeMaxRequiredFormat = "invalid %s: TcpOptions SourcePortRange Max may not be empty" + tcpSourcePortRangeMinRequiredFormat = "invalid %s: TcpOptions SourcePortRange Min may not be empty" + icmpTypeRequiredFormat = "invalid %s: IcmpOptions Type may not be empty" + destinationRequiredFormat = "invalid %s: Destination may not be empty" + sourceRequiredFormat = "invalid %s: Source may not be empty" + protocolRequiredFormat = "invalid %s: Protocol may not be empty" + invalidCIDRFormatFormat = "invalid %s: CIDR format" + apiServerBackendSetNameRegex = `^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$` + supportedSecondaryListenerPort int32 = 9345 ) // invalidNameRegex is a broad regex used to validate allows names in OCI @@ -184,6 +185,14 @@ func validateAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) continue } + if !isSupportedAPIServerListenerPort(port, apiServerPort) { + allErrs = append(allErrs, field.Invalid( + portPath, + port, + fmt.Sprintf("must be one of %s to avoid exposing arbitrary control-plane ports", supportedAPIServerListenerPortsString(apiServerPort)), + )) + continue + } } port, known := effectiveAPIServerListenerPort(backendSet.ListenerPort, apiServerPort) @@ -225,6 +234,40 @@ func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) ( return *apiServerPort, true } +func isSupportedAPIServerListenerPort(port int32, apiServerPort *int32) bool { + for _, allowedPort := range supportedAPIServerListenerPorts(apiServerPort) { + if port == allowedPort { + return true + } + } + return false +} + +func supportedAPIServerListenerPorts(apiServerPort *int32) []int32 { + ports := []int32{ociutil.DefaultAPIServerPort} + if apiServerPort != nil && *apiServerPort != ociutil.DefaultAPIServerPort { + ports = append(ports, *apiServerPort) + } + if supportedSecondaryListenerPort != ociutil.DefaultAPIServerPort { + ports = append(ports, supportedSecondaryListenerPort) + } + return ports +} + +func supportedAPIServerListenerPortsString(apiServerPort *int32) string { + values := supportedAPIServerListenerPorts(apiServerPort) + parts := make([]string, 0, len(values)) + seen := map[int32]struct{}{} + for _, value := range values { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + parts = append(parts, fmt.Sprintf("%d", value)) + } + return strings.Join(parts, ", ") +} + // validateVCNCIDR validates the CIDR of a VNC. func validateVCNCIDR(vncCIDR string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta2/validator_backendsets_test.go b/api/v1beta2/validator_backendsets_test.go index 3693c0071..952932394 100644 --- a/api/v1beta2/validator_backendsets_test.go +++ b/api/v1beta2/validator_backendsets_test.go @@ -126,6 +126,31 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { } }) + t.Run("rejects unsupported listener ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a", ListenerPort: int32Ptr(10250)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for unsupported listener port") + } + if !strings.Contains(errs[0].Error(), "must be one of 6443, 9345") { + t.Fatalf("expected supported-port guidance in error, got %q", errs[0].Error()) + } + }) + t.Run("rejects multiple omitted listener ports even when cluster API server port is unknown", func(t *testing.T) { errs := ValidateNetworkSpec( OCIClusterSubnetRoles, @@ -226,6 +251,33 @@ func TestValidateNetworkSpec_BackendSets(t *testing.T) { } }) + t.Run("rejects unsupported listener port when cluster API server port is provided", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(10250)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for unsupported listener port") + } + if !strings.Contains(errs[0].Error(), "must be one of 6443, 7443, 9345") { + t.Fatalf("expected supported-port guidance in error, got %q", errs[0].Error()) + } + }) + t.Run("accepts single omitted listener port", func(t *testing.T) { errs := ValidateNetworkSpec( OCIClusterSubnetRoles, From 1104a4c9e7828ab6eab3f7d640bb5fc3d101cf11 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 18:43:18 -0400 Subject: [PATCH 17/31] Handle concurrent LB resource creation --- cloud/scope/lb_resource_reconcile_errors.go | 22 +++ cloud/scope/load_balancer_reconciler.go | 8 + cloud/scope/load_balancer_reconciler_test.go | 93 ++++++++++++ .../scope/network_load_balancer_reconciler.go | 8 + .../network_load_balancer_reconciler_test.go | 141 ++++++++++++++++++ 5 files changed, 272 insertions(+) create mode 100644 cloud/scope/lb_resource_reconcile_errors.go diff --git a/cloud/scope/lb_resource_reconcile_errors.go b/cloud/scope/lb_resource_reconcile_errors.go new file mode 100644 index 000000000..13f9bda59 --- /dev/null +++ b/cloud/scope/lb_resource_reconcile_errors.go @@ -0,0 +1,22 @@ +package scope + +import ( + "net/http" + "strings" + + "github.com/oracle/oci-go-sdk/v65/common" +) + +func isAlreadyExistsOCIError(err error) bool { + serviceErr, ok := common.IsServiceError(err) + if !ok { + return false + } + if serviceErr.GetHTTPStatusCode() == http.StatusConflict { + return true + } + + code := strings.ToLower(serviceErr.GetCode()) + message := strings.ToLower(serviceErr.GetMessage()) + return strings.Contains(code, "already") || strings.Contains(message, "already exists") +} diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index e34e60de7..1a79caeed 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -168,6 +168,10 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu }, }) if err != nil { + if isAlreadyExistsOCIError(err) { + s.Logger.Info("LB backend set already exists during reconcile; continuing", "backendSet", name) + continue + } return errors.Wrapf(err, "failed to create lb backend set %q", name) } if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { @@ -210,6 +214,10 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu }, }) if err != nil { + if isAlreadyExistsOCIError(err) { + s.Logger.Info("LB listener already exists during reconcile; continuing", "listener", name) + continue + } return errors.Wrapf(err, "failed to create lb listener %q", name) } if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { diff --git a/cloud/scope/load_balancer_reconciler_test.go b/cloud/scope/load_balancer_reconciler_test.go index 3829679ef..b5763c032 100644 --- a/cloud/scope/load_balancer_reconciler_test.go +++ b/cloud/scope/load_balancer_reconciler_test.go @@ -1095,3 +1095,96 @@ func TestReconcileLBResources_CreatesBackendSetsBeforeListeners(t *testing.T) { g.Expect(cs.reconcileLBResources(context.Background(), desiredLB)).To(Succeed()) } + +func TestReconcileLBResources_IgnoresAlreadyExistsOnCreate(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + lbClient := mock_lb.NewMockLoadBalancerClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("lb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + LoadBalancerClient: lbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{OCICluster: ociCluster}, + Client: client, + }) + g.Expect(err).To(BeNil()) + + secondaryPort := int32(9345) + desiredLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "apiserver-lb-backendset-2", ListenerPort: &secondaryPort}, + }, + }, + } + secondaryListenerName := desiredAPIServerListenerName(1, 2, "apiserver-lb-backendset-2") + + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })).Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + }, + }, + }, nil) + + lbClient.EXPECT().CreateBackendSet(gomock.Any(), gomock.Eq(loadbalancer.CreateBackendSetRequest{ + LoadBalancerId: common.String("lb-id"), + CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ + Name: common.String("apiserver-lb-backendset-2"), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthCheckerDetails{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + })).Return(loadbalancer.CreateBackendSetResponse{}, mockServiceError{ + status: 409, + code: "Conflict", + message: "backend set already exists", + }) + lbClient.EXPECT().CreateListener(gomock.Any(), gomock.Eq(loadbalancer.CreateListenerRequest{ + LoadBalancerId: common.String("lb-id"), + CreateListenerDetails: loadbalancer.CreateListenerDetails{ + Name: common.String(secondaryListenerName), + DefaultBackendSetName: common.String("apiserver-lb-backendset-2"), + Port: common.Int(9345), + Protocol: common.String("TCP"), + }, + })).Return(loadbalancer.CreateListenerResponse{}, mockServiceError{ + status: 409, + code: "Conflict", + message: "listener already exists", + }) + + g.Expect(cs.reconcileLBResources(context.Background(), desiredLB)).To(Succeed()) +} diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index 5ff53983a..e691a71fd 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -178,6 +178,10 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc }, }) if err != nil { + if isAlreadyExistsOCIError(err) { + s.Logger.Info("NLB listener already exists during reconcile; continuing", "listener", name) + continue + } return errors.Wrapf(err, "failed to create nlb listener %q", name) } if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { @@ -236,6 +240,10 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc }, }) if err != nil { + if isAlreadyExistsOCIError(err) { + s.Logger.Info("NLB backend set already exists during reconcile; continuing", "backendSet", name) + continue + } return errors.Wrapf(err, "failed to create nlb backend set %q", name) } if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { diff --git a/cloud/scope/network_load_balancer_reconciler_test.go b/cloud/scope/network_load_balancer_reconciler_test.go index ca962e25f..aa15eb4dd 100644 --- a/cloud/scope/network_load_balancer_reconciler_test.go +++ b/cloud/scope/network_load_balancer_reconciler_test.go @@ -35,6 +35,40 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +type extendedMockNLBClient struct { + *mock_nlb.MockNetworkLoadBalancerClient + createBackendSetFn func(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) + updateBackendSetFn func(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) + deleteBackendSetFn func(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) + createListenerFn func(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) + updateListenerFn func(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) + deleteListenerFn func(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) +} + +func (c *extendedMockNLBClient) CreateBackendSet(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) { + return c.createBackendSetFn(ctx, request) +} + +func (c *extendedMockNLBClient) UpdateBackendSet(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) { + return c.updateBackendSetFn(ctx, request) +} + +func (c *extendedMockNLBClient) DeleteBackendSet(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) { + return c.deleteBackendSetFn(ctx, request) +} + +func (c *extendedMockNLBClient) CreateListener(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) { + return c.createListenerFn(ctx, request) +} + +func (c *extendedMockNLBClient) UpdateListener(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) { + return c.updateListenerFn(ctx, request) +} + +func (c *extendedMockNLBClient) DeleteListener(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) { + return c.deleteListenerFn(ctx, request) +} + func TestNLBReconciliation(t *testing.T) { var ( cs *ClusterScope @@ -925,6 +959,113 @@ func TestNLBReconciliation(t *testing.T) { } } +func TestReconcileNLBResources_IgnoresAlreadyExistsOnCreate(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + baseClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + nlbClient := &extendedMockNLBClient{ + MockNetworkLoadBalancerClient: baseClient, + createBackendSetFn: func(_ context.Context, request networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) { + if request.CreateBackendSetDetails.Name == nil || *request.CreateBackendSetDetails.Name != "rollout-set" { + t.Fatalf("unexpected backend set create request: %#v", request) + } + return networkloadbalancer.CreateBackendSetResponse{}, mockServiceError{ + status: 409, + code: "Conflict", + message: "backend set already exists", + } + }, + updateBackendSetFn: func(_ context.Context, _ networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) { + t.Fatalf("unexpected backend set update") + return networkloadbalancer.UpdateBackendSetResponse{}, nil + }, + deleteBackendSetFn: func(_ context.Context, _ networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) { + t.Fatalf("unexpected backend set delete") + return networkloadbalancer.DeleteBackendSetResponse{}, nil + }, + createListenerFn: func(_ context.Context, request networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) { + if request.CreateListenerDetails.Name == nil || *request.CreateListenerDetails.Name != desiredAPIServerListenerName(1, 2, "rollout-set") { + t.Fatalf("unexpected listener create request: %#v", request) + } + return networkloadbalancer.CreateListenerResponse{}, mockServiceError{ + status: 409, + code: "Conflict", + message: "listener already exists", + } + }, + updateListenerFn: func(_ context.Context, _ networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) { + t.Fatalf("unexpected listener update") + return networkloadbalancer.UpdateListenerResponse{}, nil + }, + deleteListenerFn: func(_ context.Context, _ networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) { + t.Fatalf("unexpected listener delete") + return networkloadbalancer.DeleteListenerResponse{}, nil + }, + } + + client := fake.NewClientBuilder().Build() + ociClusterAccessor := OCISelfManagedCluster{ + &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + }, + } + ociClusterAccessor.OCICluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociClusterAccessor.OCICluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("nlb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + NetworkLoadBalancerClient: nlbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: ociClusterAccessor, + Client: client, + }) + g.Expect(err).To(BeNil()) + + secondaryPort := int32(9345) + desiredNLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "rollout-set", ListenerPort: &secondaryPort}, + }, + }, + } + + baseClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + }, + }, + }, nil) + + g.Expect(cs.reconcileNLBResources(context.Background(), desiredNLB)).To(Succeed()) +} + func TestNLBDeletion(t *testing.T) { var ( cs *ClusterScope From 308773e5e59cdf5d41d153c0717b8807266a9c39 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 18:48:05 -0400 Subject: [PATCH 18/31] Refresh LB state before stale backend deletion --- cloud/scope/load_balancer_reconciler.go | 10 ++ cloud/scope/load_balancer_reconciler_test.go | 104 ++++++++++++++ .../scope/network_load_balancer_reconciler.go | 10 ++ .../network_load_balancer_reconciler_test.go | 136 ++++++++++++++++++ 4 files changed, 260 insertions(+) diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index 1a79caeed..1752e56df 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -245,6 +245,7 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu } } } + staleListenerDeleted := false for name := range actualResp.LoadBalancer.Listeners { if _, keep := desiredListeners[name]; keep { continue @@ -259,6 +260,15 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting delete listener %q", name) } + staleListenerDeleted = true + } + if staleListenerDeleted { + actualResp, err = s.LoadBalancerClient.GetLoadBalancer(ctx, loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: lbID, + }) + if err != nil { + return errors.Wrap(err, "failed to refresh lb after stale listener reconciliation") + } } for name, actual := range actualResp.LoadBalancer.BackendSets { if _, keep := desiredBackendSets[name]; keep { diff --git a/cloud/scope/load_balancer_reconciler_test.go b/cloud/scope/load_balancer_reconciler_test.go index b5763c032..dd71eb2bd 100644 --- a/cloud/scope/load_balancer_reconciler_test.go +++ b/cloud/scope/load_balancer_reconciler_test.go @@ -1188,3 +1188,107 @@ func TestReconcileLBResources_IgnoresAlreadyExistsOnCreate(t *testing.T) { g.Expect(cs.reconcileLBResources(context.Background(), desiredLB)).To(Succeed()) } + +func TestReconcileLBResources_RefreshesBeforeDeletingStaleBackendSets(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + lbClient := mock_lb.NewMockLoadBalancerClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("lb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + LoadBalancerClient: lbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{OCICluster: ociCluster}, + Client: client, + }) + g.Expect(err).To(BeNil()) + + desiredLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + }, + }, + } + + gomock.InOrder( + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })).Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + "stale-listener": {Name: common.String("stale-listener")}, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + "stale-set": {Name: common.String("stale-set"), Backends: []loadbalancer.Backend{}}, + }, + }, + }, nil), + lbClient.EXPECT().DeleteListener(gomock.Any(), gomock.Eq(loadbalancer.DeleteListenerRequest{ + LoadBalancerId: common.String("lb-id"), + ListenerName: common.String("stale-listener"), + })).Return(loadbalancer.DeleteListenerResponse{OpcWorkRequestId: common.String("wr-delete-listener")}, nil), + lbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(loadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-delete-listener"), + })).Return(loadbalancer.GetWorkRequestResponse{ + WorkRequest: loadbalancer.WorkRequest{LifecycleState: loadbalancer.WorkRequestLifecycleStateSucceeded}, + }, nil), + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })).Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + "stale-set": { + Name: common.String("stale-set"), + Backends: []loadbalancer.Backend{ + {Name: common.String("10.0.0.4:6443")}, + }, + }, + }, + }, + }, nil), + ) + + g.Expect(cs.reconcileLBResources(context.Background(), desiredLB)).To(Succeed()) +} diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index e691a71fd..3d9d30fd5 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -209,6 +209,7 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc } } } + staleListenerDeleted := false for name := range actualResp.NetworkLoadBalancer.Listeners { if _, keep := desiredListeners[name]; keep { continue @@ -223,6 +224,15 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting delete listener %q", name) } + staleListenerDeleted = true + } + if staleListenerDeleted { + actualResp, err = s.NetworkLoadBalancerClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: nlbID, + }) + if err != nil { + return errors.Wrap(err, "failed to refresh nlb after stale listener reconciliation") + } } for name, desired := range desiredBackendSets { diff --git a/cloud/scope/network_load_balancer_reconciler_test.go b/cloud/scope/network_load_balancer_reconciler_test.go index aa15eb4dd..a4c3659e0 100644 --- a/cloud/scope/network_load_balancer_reconciler_test.go +++ b/cloud/scope/network_load_balancer_reconciler_test.go @@ -1066,6 +1066,142 @@ func TestReconcileNLBResources_IgnoresAlreadyExistsOnCreate(t *testing.T) { g.Expect(cs.reconcileNLBResources(context.Background(), desiredNLB)).To(Succeed()) } +func TestReconcileNLBResources_RefreshesBeforeDeletingStaleBackendSets(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + baseClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + nlbClient := &extendedMockNLBClient{ + MockNetworkLoadBalancerClient: baseClient, + createBackendSetFn: func(_ context.Context, _ networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) { + t.Fatalf("unexpected backend set create") + return networkloadbalancer.CreateBackendSetResponse{}, nil + }, + updateBackendSetFn: func(_ context.Context, _ networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) { + t.Fatalf("unexpected backend set update") + return networkloadbalancer.UpdateBackendSetResponse{}, nil + }, + deleteBackendSetFn: func(_ context.Context, _ networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) { + t.Fatalf("unexpected backend set delete") + return networkloadbalancer.DeleteBackendSetResponse{}, nil + }, + createListenerFn: func(_ context.Context, _ networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) { + t.Fatalf("unexpected listener create") + return networkloadbalancer.CreateListenerResponse{}, nil + }, + updateListenerFn: func(_ context.Context, _ networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) { + t.Fatalf("unexpected listener update") + return networkloadbalancer.UpdateListenerResponse{}, nil + }, + deleteListenerFn: func(_ context.Context, request networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) { + if request.ListenerName == nil || *request.ListenerName != "stale-listener" { + t.Fatalf("unexpected listener delete request: %#v", request) + } + return networkloadbalancer.DeleteListenerResponse{OpcWorkRequestId: common.String("wr-delete-listener")}, nil + }, + } + + client := fake.NewClientBuilder().Build() + ociClusterAccessor := OCISelfManagedCluster{ + &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + }, + } + ociClusterAccessor.OCICluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociClusterAccessor.OCICluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("nlb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + NetworkLoadBalancerClient: nlbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: ociClusterAccessor, + Client: client, + }) + g.Expect(err).To(BeNil()) + + desiredNLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + }, + }, + } + + gomock.InOrder( + baseClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + "stale-listener": {Name: common.String("stale-listener")}, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + "stale-set": {Name: common.String("stale-set"), Backends: []networkloadbalancer.Backend{}}, + }, + }, + }, nil), + baseClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-delete-listener"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil), + baseClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + "stale-set": { + Name: common.String("stale-set"), + Backends: []networkloadbalancer.Backend{ + {Name: common.String("backend-1")}, + }, + }, + }, + }, + }, nil), + ) + + g.Expect(cs.reconcileNLBResources(context.Background(), desiredNLB)).To(Succeed()) +} + func TestNLBDeletion(t *testing.T) { var ( cs *ClusterScope From 27bf51fe59503d25adb2c725311ee706a88b5146 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 18:52:33 -0400 Subject: [PATCH 19/31] Fix CAPOCI E2E script and Argo auth drift --- argo/capoci-e2e-workflow.yaml | 18 ++++++++++-------- argo/capoci-e2e-workflowtemplate.yaml | 18 ++++++++++-------- scripts/ci-e2e.sh | 26 +++++++++++++++++--------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/argo/capoci-e2e-workflow.yaml b/argo/capoci-e2e-workflow.yaml index 903f26814..50c22b2a0 100644 --- a/argo/capoci-e2e-workflow.yaml +++ b/argo/capoci-e2e-workflow.yaml @@ -14,11 +14,10 @@ # -p oci_alternative_region_image_id=ocid1.image.oc1..example \ # -p registry=ghcr.io/your-org \ # -p use_instance_principal=true \ -# -p use_instance_principal_b64=dHJ1ZQ== \ # -p oci_ssh_key="ssh-rsa AAAA... your-key" # # Notes: -# - USE_INSTANCE_PRINCIPAL_B64 commonly needs "dHJ1ZQ==" for true, "ZmFsc2U=" for false (base64). +# - This workflow supports instance principal for OCIR login and the E2E run. # - If OCI_SSH_KEY is omitted, the script will generate a temporary key. # - REGISTRY defaults to ghcr.io/oracle; override if pushing to your org's registry. # - git_ref defaults to main @@ -82,10 +81,7 @@ spec: - name: exp_oke value: "false" - name: use_instance_principal - value: "false" - # Base64 of "true" or "false". Common values: true=dHJ1ZQ==, false=ZmFsc2U= - - name: use_instance_principal_b64 - value: "dHJ1ZQ==" + value: "true" - name: oci_alternative_region value: "us-sanjose-1" @@ -211,7 +207,7 @@ spec: - name: USE_INSTANCE_PRINCIPAL value: "{{workflow.parameters.use_instance_principal}}" - name: USE_INSTANCE_PRINCIPAL_B64 - value: "{{workflow.parameters.use_instance_principal_b64}}" + value: "" # Test runner config - name: GINKGO_NODES @@ -237,7 +233,7 @@ spec: - name: OCIR_REGION value: "{{workflow.parameters.ocir_region}}" - name: OCIR_REGION_SHORT_CODE - value: "{{workflow.parameters.ocir_region}}" + value: "{{workflow.parameters.ocir_region_short_code}}" - name: KIND_CGROUP_DRIVER value: "cgroupfs" @@ -288,6 +284,12 @@ spec: oci --version || true + if [ "${USE_INSTANCE_PRINCIPAL}" != "true" ]; then + echo "Argo CAPOCI E2E only supports instance principal auth for OCIR login and suite execution." >&2 + exit 1 + fi + export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "${USE_INSTANCE_PRINCIPAL}" | base64 | tr -d '\n')" + # Setup OCIR token and login echo "Generating the ocir token" echo "https://{{workflow.parameters.ocir_region}}.ocir.io/20180419/docker/token" diff --git a/argo/capoci-e2e-workflowtemplate.yaml b/argo/capoci-e2e-workflowtemplate.yaml index aae7e176a..ac949673c 100644 --- a/argo/capoci-e2e-workflowtemplate.yaml +++ b/argo/capoci-e2e-workflowtemplate.yaml @@ -25,11 +25,10 @@ # # Run a workflow from this template (override any defaults with -p): # argo submit --from workflowtemplate/capoci-e2e -n argo \ -# -p use_instance_principal=true \ -# -p use_instance_principal_b64=dHJ1ZQ== +# -p use_instance_principal=true # # Notes: -# - USE_INSTANCE_PRINCIPAL_B64 commonly needs "dHJ1ZQ==" for true, "ZmFsc2U=" for false (base64). +# - This workflow template supports instance principal for OCIR login and the E2E run. # - If OCI_SSH_KEY is omitted, the script will generate a temporary key. # - REGISTRY defaults to ghcr.io/oracle; override if pushing to your org's registry. # - git_ref defaults to main @@ -112,10 +111,7 @@ spec: - name: exp_oke value: "false" - name: use_instance_principal - value: "false" - # Base64 of "true" or "false". Common values: true=dHJ1ZQ==, false=ZmFsc2U= - - name: use_instance_principal_b64 - value: "dHJ1ZQ==" + value: "true" - name: oci_alternative_region value: "us-sanjose-1" @@ -247,7 +243,7 @@ spec: - name: USE_INSTANCE_PRINCIPAL value: "{{workflow.parameters.use_instance_principal}}" - name: USE_INSTANCE_PRINCIPAL_B64 - value: "{{workflow.parameters.use_instance_principal_b64}}" + value: "" # Test runner config - name: GINKGO_NODES @@ -329,6 +325,12 @@ spec: oci --version || true + if [ "${USE_INSTANCE_PRINCIPAL}" != "true" ]; then + echo "Argo CAPOCI E2E only supports instance principal auth for OCIR login and suite execution." >&2 + exit 1 + fi + export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "${USE_INSTANCE_PRINCIPAL}" | base64 | tr -d '\n')" + # Setup OCIR token and login using instance principal echo "Generating the ocir token" echo "https://{{workflow.parameters.ocir_region}}.ocir.io/20180419/docker/token" diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 571f01b3b..1503b1d00 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -20,20 +20,28 @@ source "${REPO_ROOT}/hack/ensure-kubectl.sh" # shellcheck source=hack/ensure-tags.sh source "${REPO_ROOT}/hack/ensure-tags.sh" +require_env() { + local name="$1" + if [ -z "${!name:-}" ]; then + echo "Environment variable ${name} is empty or not defined." >&2 + exit 1 + fi +} + # Verify the required Environment Variables are present. -: "${OCI_COMPARTMENT_ID:?Environment variable empty or not defined.}" -: "${OCI_IMAGE_ID:?Environment variable empty or not defined.}" -: "${OCI_ORACLE_LINUX_IMAGE_ID:?Environment variable empty or not defined.}" -: "${OCI_UPGRADE_IMAGE_ID:?Environment variable empty or not defined.}" -: "${OCI_ALTERNATIVE_REGION_IMAGE_ID:?Environment variable empty or not defined.}" -: "${OCI_MANAGED_NODE_IMAGE_ID:?Environment variable empty or not defined.}" -: OCI_WINDOWS_IMAGE_ID +require_env OCI_COMPARTMENT_ID +require_env OCI_IMAGE_ID +require_env OCI_ORACLE_LINUX_IMAGE_ID +require_env OCI_UPGRADE_IMAGE_ID +require_env OCI_ALTERNATIVE_REGION_IMAGE_ID +require_env OCI_MANAGED_NODE_IMAGE_ID export LOCAL_ONLY=${LOCAL_ONLY:-"true"} +export OCI_WINDOWS_IMAGE_ID="${OCI_WINDOWS_IMAGE_ID:-""}" defaultTag=$(date -u '+%Y%m%d%H%M%S') -export TAG="${defaultTag:-dev}" -export GINKGO_NODES=3 +export TAG="${TAG:-${defaultTag}}" +export GINKGO_NODES="${GINKGO_NODES:-3}" export OCI_SSH_KEY="${OCI_SSH_KEY:-""}" export OCI_CONTROL_PLANE_MACHINE_TYPE="${OCI_CONTROL_PLANE_MACHINE_TYPE:-"VM.Standard.E3.Flex"}" From 19b29fe69f613845f463c1ac43641b15869125cd Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 18:56:42 -0400 Subject: [PATCH 20/31] Verify multi-backend-set E2E LB resources --- test/e2e/cluster_test.go | 126 ++++++++++++++++++++++++++++++++++++- test/e2e/e2e_suite_test.go | 6 +- 2 files changed, 130 insertions(+), 2 deletions(-) diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index a410992eb..d85a52fa5 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -31,9 +31,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" infrastructurev1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1" + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" "github.com/oracle/cluster-api-provider-oci/cloud/scope" "github.com/oracle/oci-go-sdk/v65/common" "github.com/oracle/oci-go-sdk/v65/core" + "github.com/oracle/oci-go-sdk/v65/loadbalancer" "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -176,6 +178,7 @@ var _ = Describe("Workload cluster creation", func() { WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), }, result) + verifyAdditionalAPIServerBackendSetResources(ctx, specName, namespace.Name, clusterName) }) It("Multiple API server backend sets on LB [PRBlocking]", func() { @@ -198,6 +201,7 @@ var _ = Describe("Workload cluster creation", func() { WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), }, result) + verifyAdditionalAPIServerBackendSetResources(ctx, specName, namespace.Name, clusterName) }) It("Antrea as CNI - With 1 control-plane nodes and 1 worker nodes [DailyTests]", func() { @@ -918,8 +922,128 @@ func validateVnicNSG(ctx context.Context, clusterName string, nameSpace string, Expect(vnic.NsgIds[0]).To(Equal(*nsgId)) } } - Expect(exists).To(Equal(true)) } + Expect(exists).To(Equal(true)) +} + +func verifyAdditionalAPIServerBackendSetResources(ctx context.Context, specName, namespaceName, clusterName string) { + Eventually(func() error { + k8sClient := bootstrapClusterProxy.GetClient() + ociCluster := &infrastructurev1beta2.OCICluster{} + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespaceName, Name: clusterName}, ociCluster); err != nil { + return err + } + lbID := ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId + if lbID == nil || *lbID == "" { + return fmt.Errorf("api server load balancer ID not yet assigned") + } + + expectedBackendSets := ociCluster.Spec.NetworkSpec.APIServerLB.NLBSpec.CanonicalBackendSets() + if len(expectedBackendSets) < 2 { + return fmt.Errorf("expected multiple API server backend sets, got %d", len(expectedBackendSets)) + } + + defaultPort := ociCluster.Spec.ControlPlaneEndpoint.Port + if defaultPort == 0 { + defaultPort = 6443 + } + + switch ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerType { + case infrastructurev1beta2.LoadBalancerTypeLB: + return verifyAPIServerLBBackendSets(ctx, *lbID, defaultPort, expectedBackendSets) + case "", infrastructurev1beta2.LoadBalancerTypeNLB: + return verifyAPIServerNLBBackendSets(ctx, *lbID, defaultPort, expectedBackendSets) + default: + return fmt.Errorf("unsupported api server load balancer type %q", ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerType) + } + }, e2eConfig.GetIntervals(specName, "wait-control-plane")...).Should(Succeed()) +} + +func verifyAPIServerNLBBackendSets(ctx context.Context, lbID string, defaultPort int32, expectedBackendSets []infrastructurev1beta2.NLBBackendSet) error { + resp, err := lbClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String(lbID), + }) + if err != nil { + return err + } + + if len(resp.NetworkLoadBalancer.BackendSets) != len(expectedBackendSets) { + return fmt.Errorf("expected %d nlb backend sets, got %d", len(expectedBackendSets), len(resp.NetworkLoadBalancer.BackendSets)) + } + if len(resp.NetworkLoadBalancer.Listeners) != len(expectedBackendSets) { + return fmt.Errorf("expected %d nlb listeners, got %d", len(expectedBackendSets), len(resp.NetworkLoadBalancer.Listeners)) + } + + for _, backendSet := range expectedBackendSets { + if _, ok := resp.NetworkLoadBalancer.BackendSets[backendSet.Name]; !ok { + return fmt.Errorf("nlb backend set %q not found", backendSet.Name) + } + if !hasExpectedNLBListener(resp.NetworkLoadBalancer.Listeners, backendSet.Name, expectedAPIServerListenerPort(defaultPort, backendSet)) { + return fmt.Errorf("nlb listener for backend set %q on port %d not found", backendSet.Name, expectedAPIServerListenerPort(defaultPort, backendSet)) + } + } + + return nil +} + +func verifyAPIServerLBBackendSets(ctx context.Context, lbID string, defaultPort int32, expectedBackendSets []infrastructurev1beta2.NLBBackendSet) error { + resp, err := lbaasClient.GetLoadBalancer(ctx, loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String(lbID), + }) + if err != nil { + return err + } + + if len(resp.LoadBalancer.BackendSets) != len(expectedBackendSets) { + return fmt.Errorf("expected %d lb backend sets, got %d", len(expectedBackendSets), len(resp.LoadBalancer.BackendSets)) + } + if len(resp.LoadBalancer.Listeners) != len(expectedBackendSets) { + return fmt.Errorf("expected %d lb listeners, got %d", len(expectedBackendSets), len(resp.LoadBalancer.Listeners)) + } + + for _, backendSet := range expectedBackendSets { + if _, ok := resp.LoadBalancer.BackendSets[backendSet.Name]; !ok { + return fmt.Errorf("lb backend set %q not found", backendSet.Name) + } + if !hasExpectedLBListener(resp.LoadBalancer.Listeners, backendSet.Name, expectedAPIServerListenerPort(defaultPort, backendSet)) { + return fmt.Errorf("lb listener for backend set %q on port %d not found", backendSet.Name, expectedAPIServerListenerPort(defaultPort, backendSet)) + } + } + + return nil +} + +func hasExpectedNLBListener(listeners map[string]networkloadbalancer.Listener, backendSetName string, port int32) bool { + for _, listener := range listeners { + if listener.DefaultBackendSetName == nil || *listener.DefaultBackendSetName != backendSetName { + continue + } + if listener.Port == nil || int32(*listener.Port) != port { + continue + } + return true + } + return false +} + +func hasExpectedLBListener(listeners map[string]loadbalancer.Listener, backendSetName string, port int32) bool { + for _, listener := range listeners { + if listener.DefaultBackendSetName == nil || *listener.DefaultBackendSetName != backendSetName { + continue + } + if listener.Port == nil || int32(*listener.Port) != port { + continue + } + return true + } + return false +} + +func expectedAPIServerListenerPort(defaultPort int32, backendSet infrastructurev1beta2.NLBBackendSet) int32 { + if backendSet.ListenerPort != nil { + return *backendSet.ListenerPort + } + return defaultPort } func validateCustomNetworkingSeclist(ctx context.Context, ociCluster infrastructurev1beta1.OCICluster, clusterName string) error { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 59fce299c..1bf4f990c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -41,6 +41,7 @@ import ( "github.com/oracle/cluster-api-provider-oci/cloud/scope" "github.com/oracle/cluster-api-provider-oci/cloud/services/compute" "github.com/oracle/cluster-api-provider-oci/cloud/services/containerengine" + lb "github.com/oracle/cluster-api-provider-oci/cloud/services/loadbalancer" nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer" "github.com/oracle/cluster-api-provider-oci/cloud/services/vcn" infrav1exp "github.com/oracle/cluster-api-provider-oci/exp/api/v1beta1" @@ -107,7 +108,8 @@ var ( vcnClient vcn.Client - lbClient nlb.NetworkLoadBalancerClient + lbClient nlb.NetworkLoadBalancerClient + lbaasClient lb.LoadBalancerClient okeClient containerengine.Client @@ -246,9 +248,11 @@ var _ = SynchronizedBeforeSuite(func() []byte { identityClient := ociClients.IdentityClient vcnClient = ociClients.VCNClient lbClient = ociClients.NetworkLoadBalancerClient + lbaasClient = ociClients.LoadBalancerClient okeClient = ociClients.ContainerEngineClient Expect(identityClient).NotTo(BeNil()) Expect(lbClient).NotTo(BeNil()) + Expect(lbaasClient).NotTo(BeNil()) req := identity.ListAvailabilityDomainsRequest{CompartmentId: common.String(os.Getenv("OCI_COMPARTMENT_ID"))} resp, err := identityClient.ListAvailabilityDomains(context.Background(), req) From 507d2690a5a04dd5bb08a8590743d098ba2753b8 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 18:59:54 -0400 Subject: [PATCH 21/31] Add CAPOCI E2E runbook and preflight --- Makefile | 4 + scripts/ci-e2e-preflight.sh | 227 ++++++++++++++++++++++++++++++ test/README.md | 266 +++++++++++++++++++++++++++--------- 3 files changed, 434 insertions(+), 63 deletions(-) create mode 100755 scripts/ci-e2e-preflight.sh diff --git a/Makefile b/Makefile index 45c8d56e7..d01ffe212 100644 --- a/Makefile +++ b/Makefile @@ -313,6 +313,10 @@ test-e2e-run: generate-e2e-templates $(GINKGO) $(ENVSUBST) ## Run e2e tests -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) +.PHONY: test-e2e-preflight +test-e2e-preflight: ## Validate CAPOCI e2e prerequisites before a full run + ./scripts/ci-e2e-preflight.sh $(E2E_PREFLIGHT_ARGS) + .PHONY: test-e2e test-e2e: ## Run e2e tests PULL_POLICY=IfNotPresent MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \ diff --git a/scripts/ci-e2e-preflight.sh b/scripts/ci-e2e-preflight.sh new file mode 100755 index 000000000..3b99e710d --- /dev/null +++ b/scripts/ci-e2e-preflight.sh @@ -0,0 +1,227 @@ +#!/bin/bash +# Copyright (c) 2021, 2022 Oracle and/or its affiliates. + +set -o errexit +set -o nounset +set -o pipefail + +MODE="local" + +usage() { + cat <<'EOF' +Usage: scripts/ci-e2e-preflight.sh [--mode local|argo] + +Validates CAPOCI E2E prerequisites before running a full suite. + +Modes: + local Validate local scripts/ci-e2e.sh execution inputs. + argo Validate Argo submission inputs and instance-principal-only auth. +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + --mode) + if [ $# -lt 2 ]; then + echo "missing value for --mode" >&2 + exit 1 + fi + MODE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +case "${MODE}" in + local|argo) + ;; + *) + echo "unsupported mode: ${MODE}" >&2 + usage >&2 + exit 1 + ;; +esac + +failures=0 + +note_ok() { + echo "OK: $1" +} + +note_fail() { + echo "FAIL: $1" >&2 + failures=$((failures + 1)) +} + +check_command() { + local name="$1" + if command -v "${name}" >/dev/null 2>&1; then + note_ok "found command ${name}" + else + note_fail "missing command ${name}" + fi +} + +check_file() { + local path="$1" + if [ -f "${path}" ]; then + note_ok "found file ${path}" + else + note_fail "missing file ${path}" + fi +} + +check_env() { + local name="$1" + if [ -n "${!name:-}" ]; then + note_ok "env ${name} is set" + else + note_fail "env ${name} is not set" + fi +} + +decode_base64() { + local value="$1" + if decoded=$(printf '%s' "${value}" | base64 --decode 2>/dev/null); then + printf '%s' "${decoded}" + return 0 + fi + if decoded=$(printf '%s' "${value}" | base64 -D 2>/dev/null); then + printf '%s' "${decoded}" + return 0 + fi + return 1 +} + +check_base64_env() { + local name="$1" + local decoded="" + if [ -z "${!name:-}" ]; then + note_fail "env ${name} is not set" + return + fi + if decoded=$(decode_base64 "${!name}"); then + if [ -n "${decoded}" ]; then + note_ok "env ${name} is set and base64 decodes" + else + note_fail "env ${name} decodes to an empty value" + fi + else + note_fail "env ${name} is not valid base64" + fi +} + +uses_instance_principal_local() { + local decoded="" + if [ -n "${USE_INSTANCE_PRINCIPAL_B64:-}" ] && decoded=$(decode_base64 "${USE_INSTANCE_PRINCIPAL_B64}"); then + [ "${decoded}" = "true" ] + return + fi + return 1 +} + +check_core_envs() { + check_env OCI_COMPARTMENT_ID + check_env OCI_IMAGE_ID + check_env OCI_ORACLE_LINUX_IMAGE_ID + check_env OCI_UPGRADE_IMAGE_ID + check_env OCI_ALTERNATIVE_REGION_IMAGE_ID + check_env OCI_MANAGED_NODE_IMAGE_ID +} + +check_focus_specific_envs() { + local focus="${GINKGO_FOCUS:-PRBlocking}" + if [[ "${focus}" == *Windows* ]]; then + check_env OCI_WINDOWS_IMAGE_ID + fi + if [[ "${focus}" == *VCNPeering* ]]; then + check_env LOCAL_DRG_ID + check_env PEER_DRG_ID + check_env PEER_REGION_NAME + check_env OCI_ALTERNATIVE_REGION + check_env EXTERNAL_VCN_ID + check_env EXTERNAL_VCN_CPE_NSG + check_env EXTERNAL_VCN_WORKER_NSG + check_env EXTERNAL_VCN_CP_NSG + check_env EXTERNAL_VCN_CPE_SUBNET + check_env EXTERNAL_VCN_WORKER_SUBNET + check_env EXTERNAL_VCN_CP_SUBNET + fi +} + +check_local_auth() { + if [ -n "${AUTH_CONFIG_DIR:-}" ]; then + echo "WARN: AUTH_CONFIG_DIR is set, but the CAPOCI E2E suite does not read AUTH_CONFIG_DIR today." >&2 + fi + + if uses_instance_principal_local; then + note_ok "local auth is configured for instance principal via USE_INSTANCE_PRINCIPAL_B64" + return + fi + + check_base64_env OCI_TENANCY_ID_B64 + check_base64_env OCI_USER_ID_B64 + check_base64_env OCI_CREDENTIALS_KEY_B64 + check_base64_env OCI_CREDENTIALS_FINGERPRINT_B64 + check_base64_env OCI_REGION_B64 + if [ -n "${OCI_CREDENTIALS_PASSPHRASE_B64:-}" ]; then + check_base64_env OCI_CREDENTIALS_PASSPHRASE_B64 + fi + + if [ -n "${OCI_SESSION_TOKEN_B64:-}" ] || [ -n "${OCI_SESSION_PRIVATE_KEY_B64:-}" ]; then + note_fail "session-token auth envs are not consumed by test/e2e/e2e_suite_test.go" + fi +} + +check_argo_auth() { + if [ "${USE_INSTANCE_PRINCIPAL:-}" = "true" ]; then + note_ok "argo auth is configured for instance principal" + else + note_fail "argo workflow currently supports only USE_INSTANCE_PRINCIPAL=true" + fi +} + +echo "Running CAPOCI E2E preflight in ${MODE} mode" + +check_file "scripts/ci-e2e.sh" +check_file "test/e2e/config/e2e_conf.yaml" + +if [ "${MODE}" = "local" ]; then + check_command go + check_command docker + check_command kind + check_command kubectl + check_command kustomize + check_command envsubst + check_command jq + check_core_envs + check_focus_specific_envs + check_local_auth +else + check_command argo + check_command kubectl + check_command jq + check_file "argo/capoci-e2e-workflow.yaml" + check_file "argo/capoci-e2e-workflowtemplate.yaml" + check_file "argo/capoci-e2e-configmap.yaml" + check_core_envs + check_focus_specific_envs + check_env REGISTRY + check_argo_auth +fi + +if [ "${failures}" -ne 0 ]; then + echo "CAPOCI E2E preflight failed with ${failures} issue(s)." >&2 + exit 1 +fi + +echo "CAPOCI E2E preflight passed." diff --git a/test/README.md b/test/README.md index 4b75b2443..5cb2b3881 100644 --- a/test/README.md +++ b/test/README.md @@ -1,66 +1,206 @@ -#Prerequisite -- Install `envsubst` -- Install `kustomize` -- If you have a running `kind` cluster, please delete the `kind` cluster - -# Export the following auth settings if user principal is used as credential - ```bash - export OCI_TENANCY_ID= - export OCI_USER_ID= - export OCI_CREDENTIALS_FINGERPRINT= - export OCI_REGION= - # if Passphrase is present - export OCI_CREDENTIALS_PASSPHRASE= - export OCI_TENANCY_ID_B64="$(echo -n "$OCI_TENANCY_ID" | base64 | tr -d '\n')" - export OCI_CREDENTIALS_FINGERPRINT_B64="$(echo -n "$OCI_CREDENTIALS_FINGERPRINT" | base64 | tr -d '\n')" - export OCI_USER_ID_B64="$(echo -n "$OCI_USER_ID" | base64 | tr -d '\n')" - export OCI_REGION_B64="$(echo -n "$OCI_REGION" | base64 | tr -d '\n')" - export OCI_CREDENTIALS_KEY_B64=$( cat | base64 | tr -d '\n' ) - # if Passphrase is present - export OCI_CREDENTIALS_PASSPHRASE_B64="$(echo -n "$OCI_CREDENTIALS_PASSPHRASE" | base64 | tr -d '\n')" - ``` - -# Export the following auth settings if instance principal has to be used - ```bash - export USE_INSTANCE_PRINCIPAL_B64="$(echo -n "true" | base64 | tr -d '\n')" - ``` - - - -# Export the following test settings - ```bash - export OCI_COMPARTMENT_ID=<> - export OCI_IMAGE_ID=<> - export OCI_UPGRADE_IMAGE_ID= - export OCI_ORACLE_LINUX_IMAGE_ID= - export OCI_MANAGED_NODE_IMAGE_ID= - ``` -# Execute the test script - ```bash - export REGISTRY="iad.ocir.io/" - export LOCAL_ONLY=false - scripts/ci-e2e.sh - ``` - -# For the VCN Peering related tests, the following parameters needs to be set - ```bash - export LOCAL_DRG_ID= - export OCI_ALTERNATIVE_REGION_IMAGE_ID= - export OCI_ALTERNATIVE_REGION= - export PEER_REGION_NAME= - export PEER_DRG_ID= - ``` - -Then runt he test script by setting the correct focus parameter like below +# CAPOCI E2E Runbook + +This is the source-of-truth runbook for CAPOCI E2E execution in this repo. + +## Current auth model + +The E2E suite currently reads auth from `test/e2e/e2e_suite_test.go`. + +- Local instance principal uses `USE_INSTANCE_PRINCIPAL_B64`. +- Local user principal uses base64-encoded env vars such as `OCI_TENANCY_ID_B64`. +- Argo currently supports only instance principal for both suite execution and OCIR login. +- `AUTH_CONFIG_DIR` is not read by the E2E suite today. + +## Default suite behavior + +- `make test-e2e` builds and pushes the manager image, then runs the E2E suite. +- `make test-e2e-run` skips the image build and push step. +- Default focus is `PRBlocking`. +- Default skip is `Bare Metal|Multi-Region|VCNPeering`. +- Default Ginkgo parallelism is `GINKGO_NODES=3`. + +## Required env vars for standard runs + +These are required for both local and Argo execution: + +```bash +export OCI_COMPARTMENT_ID= +export OCI_IMAGE_ID= +export OCI_ORACLE_LINUX_IMAGE_ID= +export OCI_UPGRADE_IMAGE_ID= +export OCI_ALTERNATIVE_REGION_IMAGE_ID= +export OCI_MANAGED_NODE_IMAGE_ID= +``` + +Optional standard inputs: + +```bash +export OCI_WINDOWS_IMAGE_ID= +export OCI_SSH_KEY="ssh-rsa AAAA..." +export TAG= +export GINKGO_NODES=3 +export GINKGO_FOCUS="PRBlocking" +export GINKGO_SKIP="Bare Metal|Multi-Region|VCNPeering" +export REGISTRY= +``` + +## Scenario-specific env vars + +Windows-focused runs also require: + +```bash +export OCI_WINDOWS_IMAGE_ID= +``` + +VCN peering runs also require: + ```bash -GINKGO_SKIP="" GINKGO_FOCUS="VCNPeering" scripts/ci-e2e.sh +export LOCAL_DRG_ID= +export PEER_DRG_ID= +export PEER_REGION_NAME= +export OCI_ALTERNATIVE_REGION= +export EXTERNAL_VCN_ID= +export EXTERNAL_VCN_CPE_NSG= +export EXTERNAL_VCN_WORKER_NSG= +export EXTERNAL_VCN_CP_NSG= +export EXTERNAL_VCN_CPE_SUBNET= +export EXTERNAL_VCN_WORKER_SUBNET= +export EXTERNAL_VCN_CP_SUBNET= ``` -For the remote VCN peering test to run, the DRG has to be created manually in management cluster VCN, -and routing rules should have been added in management cluster VCN to route the workload cluster traffic to the DRG. -Following are the steps for the same -1. Create a DRG in the region where test needs to run -2. The assumption is that the VCN on which management cluster runs is already created. Add a VCN attachment in the DRG - as explained in the doc https://docs.oracle.com/en-us/iaas/Content/Network/Tasks/managingDRGs.htm under section "Attaching a VCN to a DRG" -3. Add a routing rule in the VCN Route Rules to forward traffic to workload cluster VCN, in this case, "10.1.0.0/16" - to the DRG. This is explained in the section "To route a subnet's traffic to a DRG" in the above doc. +## Local execution + +### 1. Validate prerequisites + +Run the preflight before a full suite: + +```bash +make test-e2e-preflight +``` + +The local preflight checks: + +- required tools: `go`, `docker`, `kind`, `kubectl`, `kustomize`, `envsubst`, `jq` +- required OCI image inputs +- scenario-specific env vars inferred from `GINKGO_FOCUS` +- whether auth is configured for the current suite + +### 2. Configure auth + +For local instance principal: + +```bash +export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "true" | base64 | tr -d '\n')" +``` + +For local user principal: + +```bash +export OCI_TENANCY_ID= +export OCI_USER_ID= +export OCI_CREDENTIALS_FINGERPRINT= +export OCI_REGION= +export OCI_TENANCY_ID_B64="$(printf '%s' "$OCI_TENANCY_ID" | base64 | tr -d '\n')" +export OCI_USER_ID_B64="$(printf '%s' "$OCI_USER_ID" | base64 | tr -d '\n')" +export OCI_CREDENTIALS_FINGERPRINT_B64="$(printf '%s' "$OCI_CREDENTIALS_FINGERPRINT" | base64 | tr -d '\n')" +export OCI_REGION_B64="$(printf '%s' "$OCI_REGION" | base64 | tr -d '\n')" +export OCI_CREDENTIALS_KEY_B64="$(base64 < /path/to/api-private-key.pem | tr -d '\n')" +export OCI_CREDENTIALS_PASSPHRASE_B64="$(printf '%s' "$OCI_CREDENTIALS_PASSPHRASE" | base64 | tr -d '\n')" +``` + +Notes: + +- Session-token envs are not consumed by `test/e2e/e2e_suite_test.go`. +- `AUTH_CONFIG_DIR` does not configure the E2E suite today. + +### 3. Run the suite + +Full local flow, including image build and push: + +```bash +make test-e2e-preflight +REGISTRY= scripts/ci-e2e.sh +``` + +Focused run that reuses the already-built manager image flow: + +```bash +make test-e2e-preflight +make test-e2e-run GINKGO_FOCUS="PRBlocking" GINKGO_SKIP="Bare Metal|Multi-Region|VCNPeering" +``` + +Examples: + +```bash +make test-e2e-run GINKGO_SKIP="" GINKGO_FOCUS="VCNPeering" +``` + +```bash +make test-e2e-run GINKGO_FOCUS="Conformance" E2E_ARGS='-kubetest.config-file=$(pwd)/test/e2e/data/kubetest/conformance.yaml' +``` + +Artifacts: + +- local logs and manifests are written under `_artifacts/` +- rendered config is written to `test/e2e/config/e2e_conf-envsubst.yaml` + +## Argo execution + +### 1. Validate inputs + +Run the Argo-oriented preflight: + +```bash +make test-e2e-preflight E2E_PREFLIGHT_ARGS="--mode argo" +``` + +The Argo preflight checks: + +- required tools: `argo`, `kubectl`, `jq` +- workflow/config manifests exist in the repo +- required OCI image inputs are exported +- `REGISTRY` is set for submission +- `USE_INSTANCE_PRINCIPAL=true` is selected + +### 2. Prepare the config map + +Update `argo/capoci-e2e-configmap.yaml` with the required OCI image IDs and registry, then apply it: + +```bash +kubectl apply -f argo/capoci-e2e-configmap.yaml +``` + +### 3. Submit the workflow + +Argo currently supports instance principal only: + +```bash +export USE_INSTANCE_PRINCIPAL=true +export REGISTRY= +make test-e2e-preflight E2E_PREFLIGHT_ARGS="--mode argo" + +argo submit argo/capoci-e2e-workflow.yaml \ + -p git_ref= \ + -p git_repo= \ + -p registry="${REGISTRY}" \ + -p oci_compartment_id="${OCI_COMPARTMENT_ID}" \ + -p oci_image_id="${OCI_IMAGE_ID}" \ + -p oci_oracle_linux_image_id="${OCI_ORACLE_LINUX_IMAGE_ID}" \ + -p oci_upgrade_image_id="${OCI_UPGRADE_IMAGE_ID}" \ + -p oci_alternative_region_image_id="${OCI_ALTERNATIVE_REGION_IMAGE_ID}" \ + -p oci_managed_node_image_id="${OCI_MANAGED_NODE_IMAGE_ID}" \ + -p use_instance_principal=true +``` + +You can also install and submit from the workflow template: + +```bash +kubectl apply -f argo/capoci-e2e-workflowtemplate.yaml +argo submit --from workflowtemplate/capoci-e2e -n argo -p use_instance_principal=true +``` + +Argo artifacts: + +- `report.json` +- `_artifacts_logs_only` +- `_artifacts` From 7016234b2a67b372581026582eda775af9360cb3 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 19:04:53 -0400 Subject: [PATCH 22/31] Align E2E auth with AUTH_CONFIG_DIR --- argo/capoci-e2e-workflow.yaml | 3 + argo/capoci-e2e-workflowtemplate.yaml | 3 + scripts/ci-e2e-preflight.sh | 56 ++++++++- scripts/ci-e2e.sh | 2 +- scripts/run-e2e.sh | 91 ++++++++++++++ test/README.md | 53 ++++---- test/e2e/e2e_suite_test.go | 167 +++++++++++++++++++------- 7 files changed, 306 insertions(+), 69 deletions(-) create mode 100755 scripts/run-e2e.sh diff --git a/argo/capoci-e2e-workflow.yaml b/argo/capoci-e2e-workflow.yaml index 50c22b2a0..419ebeeea 100644 --- a/argo/capoci-e2e-workflow.yaml +++ b/argo/capoci-e2e-workflow.yaml @@ -288,6 +288,9 @@ spec: echo "Argo CAPOCI E2E only supports instance principal auth for OCIR login and suite execution." >&2 exit 1 fi + export AUTH_CONFIG_DIR=/workspace/auth-config + mkdir -p "${AUTH_CONFIG_DIR}" + printf '%s' true > "${AUTH_CONFIG_DIR}/useInstancePrincipal" export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "${USE_INSTANCE_PRINCIPAL}" | base64 | tr -d '\n')" # Setup OCIR token and login diff --git a/argo/capoci-e2e-workflowtemplate.yaml b/argo/capoci-e2e-workflowtemplate.yaml index ac949673c..358afb454 100644 --- a/argo/capoci-e2e-workflowtemplate.yaml +++ b/argo/capoci-e2e-workflowtemplate.yaml @@ -329,6 +329,9 @@ spec: echo "Argo CAPOCI E2E only supports instance principal auth for OCIR login and suite execution." >&2 exit 1 fi + export AUTH_CONFIG_DIR=/workspace/auth-config + mkdir -p "${AUTH_CONFIG_DIR}" + printf '%s' true > "${AUTH_CONFIG_DIR}/useInstancePrincipal" export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "${USE_INSTANCE_PRINCIPAL}" | base64 | tr -d '\n')" # Setup OCIR token and login using instance principal diff --git a/scripts/ci-e2e-preflight.sh b/scripts/ci-e2e-preflight.sh index 3b99e710d..82ce4440f 100755 --- a/scripts/ci-e2e-preflight.sh +++ b/scripts/ci-e2e-preflight.sh @@ -120,6 +120,58 @@ check_base64_env() { fi } +check_dir_file() { + local dir="$1" + local name="$2" + if [ -f "${dir}/${name}" ]; then + note_ok "auth config file ${dir}/${name} is present" + else + note_fail "auth config file ${dir}/${name} is missing" + fi +} + +check_auth_config_dir() { + local dir="$1" + local use_instance_principal="" + local use_session_token="" + + if [ ! -d "${dir}" ]; then + note_fail "AUTH_CONFIG_DIR ${dir} does not exist" + return + fi + + check_dir_file "${dir}" "useInstancePrincipal" + if [ ! -f "${dir}/useInstancePrincipal" ]; then + return + fi + + use_instance_principal="$(tr -d '[:space:]' < "${dir}/useInstancePrincipal")" + if [ "${use_instance_principal}" = "true" ]; then + note_ok "AUTH_CONFIG_DIR selects instance principal" + return + fi + + check_dir_file "${dir}" "useSessionToken" + if [ -f "${dir}/useSessionToken" ]; then + use_session_token="$(tr -d '[:space:]' < "${dir}/useSessionToken")" + fi + + check_dir_file "${dir}" "region" + check_dir_file "${dir}" "tenancy" + check_dir_file "${dir}" "fingerprint" + if [ "${use_session_token}" = "true" ]; then + note_ok "AUTH_CONFIG_DIR selects session token auth" + check_dir_file "${dir}" "sessionToken" + check_dir_file "${dir}" "sessionPrivateKey" + return + fi + + note_ok "AUTH_CONFIG_DIR selects user principal auth" + check_dir_file "${dir}" "user" + check_dir_file "${dir}" "key" + check_dir_file "${dir}" "passphrase" +} + uses_instance_principal_local() { local decoded="" if [ -n "${USE_INSTANCE_PRINCIPAL_B64:-}" ] && decoded=$(decode_base64 "${USE_INSTANCE_PRINCIPAL_B64}"); then @@ -160,7 +212,9 @@ check_focus_specific_envs() { check_local_auth() { if [ -n "${AUTH_CONFIG_DIR:-}" ]; then - echo "WARN: AUTH_CONFIG_DIR is set, but the CAPOCI E2E suite does not read AUTH_CONFIG_DIR today." >&2 + note_ok "AUTH_CONFIG_DIR is set" + check_auth_config_dir "${AUTH_CONFIG_DIR}" + return fi if uses_instance_principal_local; then diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 1503b1d00..d2b419b61 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -62,4 +62,4 @@ if [ -z "${OCI_SSH_KEY}" ]; then export OCI_SSH_KEY fi -make test-e2e +scripts/run-e2e.sh --build-images diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh new file mode 100755 index 000000000..ed6b31de3 --- /dev/null +++ b/scripts/run-e2e.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Copyright (c) 2021, 2022 Oracle and/or its affiliates. + +set -o errexit +set -o nounset +set -o pipefail + +RUN_TARGET="test-e2e-run" +PREFLIGHT_ONLY="false" + +usage() { + cat <<'EOF' +Usage: scripts/run-e2e.sh [options] + +Supported local runner for CAPOCI E2E execution. + +Options: + --auth-config-dir AUTH_CONFIG_DIR to use for OCI auth. + --focus Override Ginkgo focus. + --skip Override Ginkgo skip. + --artifacts-dir Override artifact output directory. + --ginkgo-nodes Override Ginkgo parallelism. + --existing-cluster Reuse the current management cluster. + --skip-cleanup Preserve resources after the run. + --build-images Use make test-e2e instead of make test-e2e-run. + --preflight-only Run preflight and stop. + -h, --help Show this help message. +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + --auth-config-dir) + if [ $# -lt 2 ]; then + echo "missing value for --auth-config-dir" >&2 + exit 1 + fi + export AUTH_CONFIG_DIR="$2" + shift 2 + ;; + --focus) + export GINKGO_FOCUS="$2" + shift 2 + ;; + --skip) + export GINKGO_SKIP="$2" + shift 2 + ;; + --artifacts-dir) + export ARTIFACTS="$2" + shift 2 + ;; + --ginkgo-nodes) + export GINKGO_NODES="$2" + shift 2 + ;; + --existing-cluster) + export SKIP_CREATE_MGMT_CLUSTER=true + shift + ;; + --skip-cleanup) + export SKIP_CLEANUP=true + shift + ;; + --build-images) + RUN_TARGET="test-e2e" + shift + ;; + --preflight-only) + PREFLIGHT_ONLY="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +make test-e2e-preflight + +if [ "${PREFLIGHT_ONLY}" = "true" ]; then + exit 0 +fi + +make "${RUN_TARGET}" diff --git a/test/README.md b/test/README.md index 5cb2b3881..836e533be 100644 --- a/test/README.md +++ b/test/README.md @@ -6,10 +6,9 @@ This is the source-of-truth runbook for CAPOCI E2E execution in this repo. The E2E suite currently reads auth from `test/e2e/e2e_suite_test.go`. -- Local instance principal uses `USE_INSTANCE_PRINCIPAL_B64`. -- Local user principal uses base64-encoded env vars such as `OCI_TENANCY_ID_B64`. +- Preferred local and Argo auth uses `AUTH_CONFIG_DIR` and `cloud/config.FromDir`. +- Legacy local base64 auth envs still work as a compatibility fallback when `AUTH_CONFIG_DIR` is unset. - Argo currently supports only instance principal for both suite execution and OCIR login. -- `AUTH_CONFIG_DIR` is not read by the E2E suite today. ## Default suite behavior @@ -83,60 +82,62 @@ The local preflight checks: - required tools: `go`, `docker`, `kind`, `kubectl`, `kustomize`, `envsubst`, `jq` - required OCI image inputs - scenario-specific env vars inferred from `GINKGO_FOCUS` -- whether auth is configured for the current suite +- whether `AUTH_CONFIG_DIR` or the legacy fallback envs are configured for the current suite ### 2. Configure auth +Preferred local auth is `AUTH_CONFIG_DIR`. + For local instance principal: ```bash -export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "true" | base64 | tr -d '\n')" +mkdir -p /tmp/capoci-auth +printf '%s' true > /tmp/capoci-auth/useInstancePrincipal +export AUTH_CONFIG_DIR=/tmp/capoci-auth ``` For local user principal: ```bash -export OCI_TENANCY_ID= -export OCI_USER_ID= -export OCI_CREDENTIALS_FINGERPRINT= -export OCI_REGION= -export OCI_TENANCY_ID_B64="$(printf '%s' "$OCI_TENANCY_ID" | base64 | tr -d '\n')" -export OCI_USER_ID_B64="$(printf '%s' "$OCI_USER_ID" | base64 | tr -d '\n')" -export OCI_CREDENTIALS_FINGERPRINT_B64="$(printf '%s' "$OCI_CREDENTIALS_FINGERPRINT" | base64 | tr -d '\n')" -export OCI_REGION_B64="$(printf '%s' "$OCI_REGION" | base64 | tr -d '\n')" -export OCI_CREDENTIALS_KEY_B64="$(base64 < /path/to/api-private-key.pem | tr -d '\n')" -export OCI_CREDENTIALS_PASSPHRASE_B64="$(printf '%s' "$OCI_CREDENTIALS_PASSPHRASE" | base64 | tr -d '\n')" +mkdir -p /tmp/capoci-auth +printf '%s' false > /tmp/capoci-auth/useInstancePrincipal +printf '%s' false > /tmp/capoci-auth/useSessionToken +printf '%s' > /tmp/capoci-auth/region +printf '%s' > /tmp/capoci-auth/tenancy +printf '%s' > /tmp/capoci-auth/user +printf '%s' > /tmp/capoci-auth/fingerprint +cp /path/to/api-private-key.pem /tmp/capoci-auth/key +printf '%s' > /tmp/capoci-auth/passphrase +export AUTH_CONFIG_DIR=/tmp/capoci-auth ``` Notes: -- Session-token envs are not consumed by `test/e2e/e2e_suite_test.go`. -- `AUTH_CONFIG_DIR` does not configure the E2E suite today. +- Legacy base64 envs still work if `AUTH_CONFIG_DIR` is unset. +- Session-token auth also works through `AUTH_CONFIG_DIR` if the directory contains `useSessionToken`, `sessionToken`, and `sessionPrivateKey`. ### 3. Run the suite -Full local flow, including image build and push: +Preferred local runner: ```bash -make test-e2e-preflight -REGISTRY= scripts/ci-e2e.sh +scripts/run-e2e.sh --auth-config-dir "${AUTH_CONFIG_DIR}" --focus "PRBlocking" ``` -Focused run that reuses the already-built manager image flow: +Full local flow, including image build and push: ```bash -make test-e2e-preflight -make test-e2e-run GINKGO_FOCUS="PRBlocking" GINKGO_SKIP="Bare Metal|Multi-Region|VCNPeering" +REGISTRY= scripts/ci-e2e.sh ``` Examples: ```bash -make test-e2e-run GINKGO_SKIP="" GINKGO_FOCUS="VCNPeering" +scripts/run-e2e.sh --auth-config-dir "${AUTH_CONFIG_DIR}" --focus "VCNPeering" --skip "" ``` ```bash -make test-e2e-run GINKGO_FOCUS="Conformance" E2E_ARGS='-kubetest.config-file=$(pwd)/test/e2e/data/kubetest/conformance.yaml' +scripts/run-e2e.sh --auth-config-dir "${AUTH_CONFIG_DIR}" --existing-cluster --artifacts-dir "$(pwd)/_artifacts-existing" ``` Artifacts: @@ -192,6 +193,8 @@ argo submit argo/capoci-e2e-workflow.yaml \ -p use_instance_principal=true ``` +The workflow writes a temporary `AUTH_CONFIG_DIR` inside the job before running `scripts/ci-e2e.sh`. + You can also install and submit from the workflow template: ```bash diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1bf4f990c..a60552112 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -193,48 +193,8 @@ var _ = SynchronizedBeforeSuite(func() []byte { e2eConfig = loadE2EConfig(configPath) bootstrapClusterProxy = NewOCIClusterProxy("bootstrap", kubeconfigPath, initScheme()) - s, useInstancePrincipalFlagSet := os.LookupEnv("USE_INSTANCE_PRINCIPAL_B64") - useInstanePrincipal := false - if useInstancePrincipalFlagSet { - useInstanePrincipalStr, err := base64.StdEncoding.DecodeString(s) - Expect(err).NotTo(HaveOccurred()) - useInstanePrincipal, err = strconv.ParseBool(string(useInstanePrincipalStr)) - Expect(err).NotTo(HaveOccurred()) - } - var ociAuthConfigProvider common.ConfigurationProvider - var err error - if useInstanePrincipal { - ociAuthConfigProvider, err = oci_config.NewConfigurationProvider(&oci_config.AuthConfig{ - UseInstancePrincipals: true, - }) - Expect(err).NotTo(HaveOccurred()) - By("Using instance principal as auth provider") - } else { - tenancyId, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_TENANCY_ID_B64")) - Expect(err).NotTo(HaveOccurred()) - userId, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_USER_ID_B64")) - Expect(err).NotTo(HaveOccurred()) - passphrase, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_CREDENTIALS_PASSPHRASE_B64")) - Expect(err).NotTo(HaveOccurred()) - key, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_CREDENTIALS_KEY_B64")) - Expect(err).NotTo(HaveOccurred()) - fingerprint, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_CREDENTIALS_FINGERPRINT_B64")) - Expect(err).NotTo(HaveOccurred()) - region, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_REGION_B64")) - Expect(err).NotTo(HaveOccurred()) - - ociAuthConfigProvider, err = oci_config.NewConfigurationProvider(&oci_config.AuthConfig{ - Region: string(region), - TenancyID: string(tenancyId), - UserID: string(userId), - PrivateKey: string(key), - Fingerprint: string(fingerprint), - Passphrase: string(passphrase), - UseInstancePrincipals: false, - }) - Expect(err).NotTo(HaveOccurred()) - By("Using user principal as auth provider") - } + ociAuthConfigProvider, err := newE2EAuthConfigProvider() + Expect(err).NotTo(HaveOccurred()) clientProvider, err := scope.NewClientProvider(scope.ClientProviderParams{OciAuthConfigProvider: ociAuthConfigProvider}) Expect(err).NotTo(HaveOccurred()) @@ -259,6 +219,129 @@ var _ = SynchronizedBeforeSuite(func() []byte { adCount = len(resp.Items) }) +func newE2EAuthConfigProvider() (common.ConfigurationProvider, error) { + if authConfigDir := os.Getenv("AUTH_CONFIG_DIR"); authConfigDir != "" { + authConfig, err := oci_config.FromDir(authConfigDir) + if err != nil { + return nil, err + } + provider, err := oci_config.NewConfigurationProvider(authConfig) + if err != nil { + return nil, err + } + switch { + case authConfig.UseInstancePrincipals: + By(fmt.Sprintf("Using auth provider from AUTH_CONFIG_DIR %q with instance principal", authConfigDir)) + case authConfig.UseSessionToken: + By(fmt.Sprintf("Using auth provider from AUTH_CONFIG_DIR %q with session token", authConfigDir)) + default: + By(fmt.Sprintf("Using auth provider from AUTH_CONFIG_DIR %q with user principal", authConfigDir)) + } + return provider, nil + } + + useInstancePrincipal, err := useInstancePrincipalFromEnv() + if err != nil { + return nil, err + } + if useInstancePrincipal { + provider, err := oci_config.NewConfigurationProvider(&oci_config.AuthConfig{ + UseInstancePrincipals: true, + }) + if err != nil { + return nil, err + } + By("Using instance principal as auth provider from legacy env vars") + return provider, nil + } + + authConfig, err := legacyBase64AuthConfigFromEnv() + if err != nil { + return nil, err + } + provider, err := oci_config.NewConfigurationProvider(authConfig) + if err != nil { + return nil, err + } + By("Using user principal as auth provider from legacy env vars") + return provider, nil +} + +func useInstancePrincipalFromEnv() (bool, error) { + if rawValue, ok := os.LookupEnv("USE_INSTANCE_PRINCIPAL"); ok && rawValue != "" { + return strconv.ParseBool(rawValue) + } + if encodedValue, ok := os.LookupEnv("USE_INSTANCE_PRINCIPAL_B64"); ok && encodedValue != "" { + decodedValue, err := base64.StdEncoding.DecodeString(encodedValue) + if err != nil { + return false, err + } + return strconv.ParseBool(string(decodedValue)) + } + return false, nil +} + +func legacyBase64AuthConfigFromEnv() (*oci_config.AuthConfig, error) { + tenancyID, err := decodeRequiredBase64Env("OCI_TENANCY_ID_B64") + if err != nil { + return nil, err + } + userID, err := decodeRequiredBase64Env("OCI_USER_ID_B64") + if err != nil { + return nil, err + } + privateKey, err := decodeRequiredBase64Env("OCI_CREDENTIALS_KEY_B64") + if err != nil { + return nil, err + } + fingerprint, err := decodeRequiredBase64Env("OCI_CREDENTIALS_FINGERPRINT_B64") + if err != nil { + return nil, err + } + region, err := decodeRequiredBase64Env("OCI_REGION_B64") + if err != nil { + return nil, err + } + passphrase, err := decodeOptionalBase64Env("OCI_CREDENTIALS_PASSPHRASE_B64") + if err != nil { + return nil, err + } + + return &oci_config.AuthConfig{ + Region: region, + TenancyID: tenancyID, + UserID: userID, + PrivateKey: privateKey, + Fingerprint: fingerprint, + Passphrase: passphrase, + UseInstancePrincipals: false, + }, nil +} + +func decodeRequiredBase64Env(name string) (string, error) { + value := os.Getenv(name) + if value == "" { + return "", fmt.Errorf("%s is not set", name) + } + decoded, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return "", err + } + return string(decoded), nil +} + +func decodeOptionalBase64Env(name string) (string, error) { + value := os.Getenv(name) + if value == "" { + return "", nil + } + decoded, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return "", err + } + return string(decoded), nil +} + // Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads). // The bootstrap cluster is shared across all the tests, so it should be deleted only after all ParallelNodes completes. // The local clusterctl repository is preserved like everything else created into the artifact folder. From ce2ae8553f75cee05bc00081ed39a06ca1b54d1f Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 19:06:26 -0400 Subject: [PATCH 23/31] Trim PRBlocking multi-backend-set E2E scope --- test/e2e/cluster_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index d85a52fa5..f3a4ea9af 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -181,7 +181,7 @@ var _ = Describe("Workload cluster creation", func() { verifyAdditionalAPIServerBackendSetResources(ctx, specName, namespace.Name, clusterName) }) - It("Multiple API server backend sets on LB [PRBlocking]", func() { + It("Multiple API server backend sets on LB [DailyTests]", func() { clusterName = getClusterName(clusterNamePrefix, "multiple-backends-lb") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ ClusterProxy: bootstrapClusterProxy, From 64f4cc9a6dd0c4715fcc4d3ff074f7a0688a5776 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 19:22:45 -0400 Subject: [PATCH 24/31] Make NLB reconcile interface explicit --- .../scope/network_load_balancer_reconciler.go | 29 ++--- .../network_load_balancer_reconciler_test.go | 105 ++++++++++++------ cloud/services/networkloadbalancer/client.go | 6 + .../mock_nlb/client_mock.go | 90 +++++++++++++++ 4 files changed, 175 insertions(+), 55 deletions(-) diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index 3d9d30fd5..3781ff94a 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -29,15 +29,6 @@ import ( clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" ) -type nlbExtendedClient interface { - CreateBackendSet(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (response networkloadbalancer.CreateBackendSetResponse, err error) - UpdateBackendSet(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (response networkloadbalancer.UpdateBackendSetResponse, err error) - DeleteBackendSet(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (response networkloadbalancer.DeleteBackendSetResponse, err error) - CreateListener(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (response networkloadbalancer.CreateListenerResponse, err error) - UpdateListener(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (response networkloadbalancer.UpdateListenerResponse, err error) - DeleteListener(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (response networkloadbalancer.DeleteListenerResponse, err error) -} - // ReconcileApiServerNLB tries to move the Network Load Balancer to the desired OCICluster Spec func (s *ClusterScope) ReconcileApiServerNLB(ctx context.Context) error { desiredApiServerNLB := s.NLBSpec() @@ -150,12 +141,6 @@ func (s *ClusterScope) UpdateNLB(ctx context.Context, nlb infrastructurev1beta2. } func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastructurev1beta2.LoadBalancer) error { - extendedClient, ok := any(s.NetworkLoadBalancerClient).(nlbExtendedClient) - if !ok { - // Unit tests use minimal mocks without listener/backend-set management methods. - s.Logger.Info("NLB client does not support extended listener/backend-set reconcile") - return nil - } nlbID := s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.LoadBalancerId actualResp, err := s.NetworkLoadBalancerClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ NetworkLoadBalancerId: nlbID, @@ -168,7 +153,7 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc for name, desired := range desiredListeners { actual, exists := actualResp.NetworkLoadBalancer.Listeners[name] if !exists { - resp, err := extendedClient.CreateListener(ctx, networkloadbalancer.CreateListenerRequest{ + resp, err := s.NetworkLoadBalancerClient.CreateListener(ctx, networkloadbalancer.CreateListenerRequest{ NetworkLoadBalancerId: nlbID, CreateListenerDetails: networkloadbalancer.CreateListenerDetails{ Name: common.String(name), @@ -192,7 +177,7 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc if !intPtrEqual(actual.Port, desired.Port) || !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || actual.Protocol != desired.Protocol { - resp, err := extendedClient.UpdateListener(ctx, networkloadbalancer.UpdateListenerRequest{ + resp, err := s.NetworkLoadBalancerClient.UpdateListener(ctx, networkloadbalancer.UpdateListenerRequest{ NetworkLoadBalancerId: nlbID, ListenerName: common.String(name), UpdateListenerDetails: networkloadbalancer.UpdateListenerDetails{ @@ -214,7 +199,7 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc if _, keep := desiredListeners[name]; keep { continue } - resp, err := extendedClient.DeleteListener(ctx, networkloadbalancer.DeleteListenerRequest{ + resp, err := s.NetworkLoadBalancerClient.DeleteListener(ctx, networkloadbalancer.DeleteListenerRequest{ NetworkLoadBalancerId: nlbID, ListenerName: common.String(name), }) @@ -238,7 +223,7 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc for name, desired := range desiredBackendSets { actual, exists := actualResp.NetworkLoadBalancer.BackendSets[name] if !exists { - resp, err := extendedClient.CreateBackendSet(ctx, networkloadbalancer.CreateBackendSetRequest{ + resp, err := s.NetworkLoadBalancerClient.CreateBackendSet(ctx, networkloadbalancer.CreateBackendSetRequest{ NetworkLoadBalancerId: nlbID, CreateBackendSetDetails: networkloadbalancer.CreateBackendSetDetails{ Name: common.String(name), @@ -263,7 +248,7 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc } if actual.HealthChecker == nil || desired.HealthChecker == nil { // Reconcile nullable health checker by updating if either side is nil. - resp, err := extendedClient.UpdateBackendSet(ctx, networkloadbalancer.UpdateBackendSetRequest{ + resp, err := s.NetworkLoadBalancerClient.UpdateBackendSet(ctx, networkloadbalancer.UpdateBackendSetRequest{ NetworkLoadBalancerId: nlbID, BackendSetName: common.String(name), UpdateBackendSetDetails: networkloadbalancer.UpdateBackendSetDetails{ @@ -289,7 +274,7 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || actual.HealthChecker.Protocol != desired.HealthChecker.Protocol || !ptr.StringEquals(actual.HealthChecker.UrlPath, ptr.ToString(desired.HealthChecker.UrlPath)) { - resp, err := extendedClient.UpdateBackendSet(ctx, networkloadbalancer.UpdateBackendSetRequest{ + resp, err := s.NetworkLoadBalancerClient.UpdateBackendSet(ctx, networkloadbalancer.UpdateBackendSetRequest{ NetworkLoadBalancerId: nlbID, BackendSetName: common.String(name), UpdateBackendSetDetails: networkloadbalancer.UpdateBackendSetDetails{ @@ -315,7 +300,7 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc if len(actual.Backends) > 0 { continue } - resp, err := extendedClient.DeleteBackendSet(ctx, networkloadbalancer.DeleteBackendSetRequest{ + resp, err := s.NetworkLoadBalancerClient.DeleteBackendSet(ctx, networkloadbalancer.DeleteBackendSetRequest{ NetworkLoadBalancerId: nlbID, BackendSetName: common.String(name), }) diff --git a/cloud/scope/network_load_balancer_reconciler_test.go b/cloud/scope/network_load_balancer_reconciler_test.go index a4c3659e0..231521de8 100644 --- a/cloud/scope/network_load_balancer_reconciler_test.go +++ b/cloud/scope/network_load_balancer_reconciler_test.go @@ -172,50 +172,52 @@ func TestNLBReconciliation(t *testing.T) { errorExpected: false, testSpecificSetup: func(clusterScope *ClusterScope, nlbClient *mock_nlb.MockNetworkLoadBalancerClient) { secondaryPort := int32(9345) + secondaryListenerName := desiredAPIServerListenerName(1, 2, "rollout-set") clusterScope.OCIClusterAccessor.GetNetworkSpec().APIServerLB.LoadBalancerId = common.String("nlb-id") clusterScope.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec.BackendSets = []infrastructurev1beta2.NLBBackendSet{ {Name: APIServerLBBackendSetName}, {Name: "rollout-set", ListenerPort: &secondaryPort}, } - nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + getReq := networkloadbalancer.GetNetworkLoadBalancerRequest{ NetworkLoadBalancerId: common.String("nlb-id"), - })). - Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ - NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ - LifecycleState: networkloadbalancer.LifecycleStateActive, - Id: common.String("nlb-id"), - FreeformTags: tags, - DefinedTags: make(map[string]map[string]interface{}), - IsPrivate: common.Bool(false), - DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), - IpAddresses: []networkloadbalancer.IpAddress{ - { - IpAddress: common.String("2.2.2.2"), - IsPublic: common.Bool(true), - }, + } + getResp := networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + LifecycleState: networkloadbalancer.LifecycleStateActive, + Id: common.String("nlb-id"), + FreeformTags: tags, + DefinedTags: make(map[string]map[string]interface{}), + IsPrivate: common.Bool(false), + DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + IpAddresses: []networkloadbalancer.IpAddress{ + { + IpAddress: common.String("2.2.2.2"), + IsPublic: common.Bool(true), }, - Listeners: map[string]networkloadbalancer.Listener{ - APIServerLBListener: { - Name: common.String(APIServerLBListener), - DefaultBackendSetName: common.String(APIServerLBBackendSetName), - Port: common.Int(6443), - Protocol: networkloadbalancer.ListenerProtocolsTcp, - }, + }, + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, }, - BackendSets: map[string]networkloadbalancer.BackendSet{ - APIServerLBBackendSetName: { - Name: common.String(APIServerLBBackendSetName), - Policy: LoadBalancerPolicy, - IsPreserveSource: common.Bool(false), - HealthChecker: &networkloadbalancer.HealthChecker{ - Port: common.Int(6443), - Protocol: networkloadbalancer.HealthCheckProtocolsHttps, - UrlPath: common.String("/healthz"), - }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), }, }, }, - }, nil) + }, + } + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(getReq)).Return(getResp, nil) nlbClient.EXPECT().UpdateNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.UpdateNetworkLoadBalancerRequest{ NetworkLoadBalancerId: common.String("nlb-id"), UpdateNetworkLoadBalancerDetails: networkloadbalancer.UpdateNetworkLoadBalancerDetails{ @@ -232,6 +234,43 @@ func TestNLBReconciliation(t *testing.T) { Status: networkloadbalancer.OperationStatusSucceeded, }, }, nil) + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(getReq)).Return(getResp, nil) + nlbClient.EXPECT().CreateListener(gomock.Any(), gomock.Eq(networkloadbalancer.CreateListenerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + CreateListenerDetails: networkloadbalancer.CreateListenerDetails{ + Name: common.String(secondaryListenerName), + DefaultBackendSetName: common.String("rollout-set"), + Port: common.Int(int(secondaryPort)), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + })).Return(networkloadbalancer.CreateListenerResponse{ + OpcWorkRequestId: common.String("wr-create-listener"), + }, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-create-listener"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{ + Status: networkloadbalancer.OperationStatusSucceeded, + }, + }, nil) + nlbClient.EXPECT().CreateBackendSet(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendSetRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + CreateBackendSetDetails: networkloadbalancer.CreateBackendSetDetails{ + Name: common.String("rollout-set"), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthCheckerDetails{Port: common.Int(6443), Protocol: networkloadbalancer.HealthCheckProtocolsHttps, UrlPath: common.String("/healthz"), ReturnCode: common.Int(200)}, + }, + })).Return(networkloadbalancer.CreateBackendSetResponse{ + OpcWorkRequestId: common.String("wr-create-backendset"), + }, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-create-backendset"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{ + Status: networkloadbalancer.OperationStatusSucceeded, + }, + }, nil) }, }, { diff --git a/cloud/services/networkloadbalancer/client.go b/cloud/services/networkloadbalancer/client.go index e9b386b60..e40c20f30 100644 --- a/cloud/services/networkloadbalancer/client.go +++ b/cloud/services/networkloadbalancer/client.go @@ -26,10 +26,16 @@ type NetworkLoadBalancerClient interface { ListNetworkLoadBalancers(ctx context.Context, request networkloadbalancer.ListNetworkLoadBalancersRequest) (response networkloadbalancer.ListNetworkLoadBalancersResponse, err error) GetNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.GetNetworkLoadBalancerRequest) (response networkloadbalancer.GetNetworkLoadBalancerResponse, err error) CreateBackend(ctx context.Context, request networkloadbalancer.CreateBackendRequest) (response networkloadbalancer.CreateBackendResponse, err error) + CreateBackendSet(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (response networkloadbalancer.CreateBackendSetResponse, err error) CreateNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.CreateNetworkLoadBalancerRequest) (response networkloadbalancer.CreateNetworkLoadBalancerResponse, err error) + CreateListener(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (response networkloadbalancer.CreateListenerResponse, err error) DeleteBackend(ctx context.Context, request networkloadbalancer.DeleteBackendRequest) (response networkloadbalancer.DeleteBackendResponse, err error) + DeleteBackendSet(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (response networkloadbalancer.DeleteBackendSetResponse, err error) + DeleteListener(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (response networkloadbalancer.DeleteListenerResponse, err error) GetWorkRequest(ctx context.Context, request networkloadbalancer.GetWorkRequestRequest) (response networkloadbalancer.GetWorkRequestResponse, err error) ListWorkRequestErrors(ctx context.Context, request networkloadbalancer.ListWorkRequestErrorsRequest) (response networkloadbalancer.ListWorkRequestErrorsResponse, err error) + UpdateBackendSet(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (response networkloadbalancer.UpdateBackendSetResponse, err error) + UpdateListener(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (response networkloadbalancer.UpdateListenerResponse, err error) UpdateNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.UpdateNetworkLoadBalancerRequest) (response networkloadbalancer.UpdateNetworkLoadBalancerResponse, err error) DeleteNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.DeleteNetworkLoadBalancerRequest) (response networkloadbalancer.DeleteNetworkLoadBalancerResponse, err error) } diff --git a/cloud/services/networkloadbalancer/mock_nlb/client_mock.go b/cloud/services/networkloadbalancer/mock_nlb/client_mock.go index 888bc29fc..747c6a545 100644 --- a/cloud/services/networkloadbalancer/mock_nlb/client_mock.go +++ b/cloud/services/networkloadbalancer/mock_nlb/client_mock.go @@ -50,6 +50,21 @@ func (mr *MockNetworkLoadBalancerClientMockRecorder) CreateBackend(ctx, request return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackend", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).CreateBackend), ctx, request) } +// CreateBackendSet mocks base method. +func (m *MockNetworkLoadBalancerClient) CreateBackendSet(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateBackendSet", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.CreateBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateBackendSet indicates an expected call of CreateBackendSet. +func (mr *MockNetworkLoadBalancerClientMockRecorder) CreateBackendSet(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackendSet", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).CreateBackendSet), ctx, request) +} + // CreateNetworkLoadBalancer mocks base method. func (m *MockNetworkLoadBalancerClient) CreateNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.CreateNetworkLoadBalancerRequest) (networkloadbalancer.CreateNetworkLoadBalancerResponse, error) { m.ctrl.T.Helper() @@ -65,6 +80,21 @@ func (mr *MockNetworkLoadBalancerClientMockRecorder) CreateNetworkLoadBalancer(c return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNetworkLoadBalancer", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).CreateNetworkLoadBalancer), ctx, request) } +// CreateListener mocks base method. +func (m *MockNetworkLoadBalancerClient) CreateListener(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateListener", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.CreateListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateListener indicates an expected call of CreateListener. +func (mr *MockNetworkLoadBalancerClientMockRecorder) CreateListener(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateListener", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).CreateListener), ctx, request) +} + // DeleteBackend mocks base method. func (m *MockNetworkLoadBalancerClient) DeleteBackend(ctx context.Context, request networkloadbalancer.DeleteBackendRequest) (networkloadbalancer.DeleteBackendResponse, error) { m.ctrl.T.Helper() @@ -80,6 +110,36 @@ func (mr *MockNetworkLoadBalancerClientMockRecorder) DeleteBackend(ctx, request return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBackend", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).DeleteBackend), ctx, request) } +// DeleteBackendSet mocks base method. +func (m *MockNetworkLoadBalancerClient) DeleteBackendSet(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBackendSet", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.DeleteBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteBackendSet indicates an expected call of DeleteBackendSet. +func (mr *MockNetworkLoadBalancerClientMockRecorder) DeleteBackendSet(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBackendSet", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).DeleteBackendSet), ctx, request) +} + +// DeleteListener mocks base method. +func (m *MockNetworkLoadBalancerClient) DeleteListener(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteListener", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.DeleteListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteListener indicates an expected call of DeleteListener. +func (mr *MockNetworkLoadBalancerClientMockRecorder) DeleteListener(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteListener", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).DeleteListener), ctx, request) +} + // DeleteNetworkLoadBalancer mocks base method. func (m *MockNetworkLoadBalancerClient) DeleteNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.DeleteNetworkLoadBalancerRequest) (networkloadbalancer.DeleteNetworkLoadBalancerResponse, error) { m.ctrl.T.Helper() @@ -169,3 +229,33 @@ func (mr *MockNetworkLoadBalancerClientMockRecorder) UpdateNetworkLoadBalancer(c mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateNetworkLoadBalancer", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).UpdateNetworkLoadBalancer), ctx, request) } + +// UpdateBackendSet mocks base method. +func (m *MockNetworkLoadBalancerClient) UpdateBackendSet(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateBackendSet", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.UpdateBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateBackendSet indicates an expected call of UpdateBackendSet. +func (mr *MockNetworkLoadBalancerClientMockRecorder) UpdateBackendSet(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBackendSet", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).UpdateBackendSet), ctx, request) +} + +// UpdateListener mocks base method. +func (m *MockNetworkLoadBalancerClient) UpdateListener(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateListener", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.UpdateListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateListener indicates an expected call of UpdateListener. +func (mr *MockNetworkLoadBalancerClientMockRecorder) UpdateListener(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateListener", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).UpdateListener), ctx, request) +} From cd477541fa1d9b0c0c0b423f073c00bf87553674 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 19:28:58 -0400 Subject: [PATCH 25/31] Deduplicate API server backend-set helpers --- api/internal/apiserverlb/backendsets.go | 153 ++++++++++++++++++++++++ api/v1beta1/backendsets_shared.go | 42 +++++++ api/v1beta1/types.go | 10 +- api/v1beta1/validator.go | 130 +------------------- api/v1beta2/backendsets_shared.go | 42 +++++++ api/v1beta2/types.go | 10 +- api/v1beta2/validator.go | 130 +------------------- 7 files changed, 241 insertions(+), 276 deletions(-) create mode 100644 api/internal/apiserverlb/backendsets.go create mode 100644 api/v1beta1/backendsets_shared.go create mode 100644 api/v1beta2/backendsets_shared.go diff --git a/api/internal/apiserverlb/backendsets.go b/api/internal/apiserverlb/backendsets.go new file mode 100644 index 000000000..42d2b67de --- /dev/null +++ b/api/internal/apiserverlb/backendsets.go @@ -0,0 +1,153 @@ +package apiserverlb + +import ( + "fmt" + "regexp" + "strings" + + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +const BackendSetNameRegex = "^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$" + +type BackendSet struct { + Name string + ListenerPort *int32 +} + +func CanonicalBackendSets[T any](configured []T, synthesize func() T) []T { + if len(configured) > 0 { + return append([]T(nil), configured...) + } + return []T{synthesize()} +} + +func ValidateBackendSets(backendSets []BackendSet, legacyConfigured bool, apiServerPort *int32, supportedSecondaryListenerPort int32, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + if len(backendSets) > 0 && legacyConfigured { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child("backendSetDetails"), + "cannot be set when backendSets is configured; move legacy backendSetDetails into backendSets", + )) + } + + nameRegex := regexp.MustCompile(BackendSetNameRegex) + seen := map[string]int{} + for i, backendSet := range backendSets { + namePath := fldPath.Child("backendSets").Index(i).Child("name") + name := strings.TrimSpace(backendSet.Name) + if name == "" { + allErrs = append(allErrs, field.Required(namePath, "must not be empty")) + continue + } + if !nameRegex.MatchString(name) { + allErrs = append(allErrs, field.Invalid( + namePath, + backendSet.Name, + "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$ (1-32 chars; alphanumeric, '-', '_')", + )) + } + if firstIdx, ok := seen[name]; ok { + allErrs = append(allErrs, field.Invalid( + namePath, + backendSet.Name, + fmt.Sprintf("duplicate backend set name, already used at backendSets[%d].name", firstIdx), + )) + continue + } + seen[name] = i + } + + usedPorts := map[int32]int{} + omittedListenerPortIndex := -1 + for i, backendSet := range backendSets { + portPath := fldPath.Child("backendSets").Index(i).Child("listenerPort") + if backendSet.ListenerPort != nil { + port := *backendSet.ListenerPort + if port < 1 || port > 65535 { + allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) + continue + } + if !IsSupportedAPIServerListenerPort(port, apiServerPort, supportedSecondaryListenerPort) { + allErrs = append(allErrs, field.Invalid( + portPath, + port, + fmt.Sprintf("must be one of %s to avoid exposing arbitrary control-plane ports", SupportedAPIServerListenerPortsString(apiServerPort, supportedSecondaryListenerPort)), + )) + continue + } + } + + port, known := EffectiveAPIServerListenerPort(backendSet.ListenerPort, apiServerPort) + if !known { + if omittedListenerPortIndex >= 0 { + allErrs = append(allErrs, field.Invalid( + portPath, + backendSet.ListenerPort, + fmt.Sprintf("multiple backend sets omit listenerPort, already omitted at backendSets[%d].listenerPort", omittedListenerPortIndex), + )) + continue + } + omittedListenerPortIndex = i + continue + } + + if firstIdx, ok := usedPorts[port]; ok { + allErrs = append(allErrs, field.Invalid( + portPath, + backendSet.ListenerPort, + fmt.Sprintf("duplicate effective listenerPort %d, already used at backendSets[%d].listenerPort", port, firstIdx), + )) + continue + } + usedPorts[port] = i + } + + return allErrs +} + +func EffectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { + if listenerPort != nil { + return *listenerPort, true + } + if apiServerPort == nil { + return 0, false + } + return *apiServerPort, true +} + +func IsSupportedAPIServerListenerPort(port int32, apiServerPort *int32, supportedSecondaryListenerPort int32) bool { + for _, allowedPort := range SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort) { + if port == allowedPort { + return true + } + } + return false +} + +func SupportedAPIServerListenerPorts(apiServerPort *int32, supportedSecondaryListenerPort int32) []int32 { + ports := []int32{ociutil.DefaultAPIServerPort} + if apiServerPort != nil && *apiServerPort != ociutil.DefaultAPIServerPort { + ports = append(ports, *apiServerPort) + } + if supportedSecondaryListenerPort != ociutil.DefaultAPIServerPort { + ports = append(ports, supportedSecondaryListenerPort) + } + return ports +} + +func SupportedAPIServerListenerPortsString(apiServerPort *int32, supportedSecondaryListenerPort int32) string { + values := SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort) + parts := make([]string, 0, len(values)) + seen := map[int32]struct{}{} + for _, value := range values { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + parts = append(parts, fmt.Sprintf("%d", value)) + } + return strings.Join(parts, ", ") +} diff --git a/api/v1beta1/backendsets_shared.go b/api/v1beta1/backendsets_shared.go new file mode 100644 index 000000000..1f4366aad --- /dev/null +++ b/api/v1beta1/backendsets_shared.go @@ -0,0 +1,42 @@ +package v1beta1 + +import ( + apiserverlb "github.com/oracle/cluster-api-provider-oci/api/internal/apiserverlb" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func canonicalAPIServerBackendSets(configured []NLBBackendSet, legacy BackendSetDetails) []NLBBackendSet { + return apiserverlb.CanonicalBackendSets(configured, func() NLBBackendSet { + return NLBBackendSet{ + Name: APIServerLBBackendSetName, + BackendSetDetails: legacy, + } + }) +} + +func validateSharedAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { + backendSets := make([]apiserverlb.BackendSet, 0, len(spec.BackendSets)) + for _, backendSet := range spec.BackendSets { + backendSets = append(backendSets, apiserverlb.BackendSet{ + Name: backendSet.Name, + ListenerPort: backendSet.ListenerPort, + }) + } + return apiserverlb.ValidateBackendSets(backendSets, spec.BackendSetDetails != (BackendSetDetails{}), apiServerPort, supportedSecondaryListenerPort, fldPath) +} + +func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { + return apiserverlb.EffectiveAPIServerListenerPort(listenerPort, apiServerPort) +} + +func isSupportedAPIServerListenerPort(port int32, apiServerPort *int32) bool { + return apiserverlb.IsSupportedAPIServerListenerPort(port, apiServerPort, supportedSecondaryListenerPort) +} + +func supportedAPIServerListenerPorts(apiServerPort *int32) []int32 { + return apiserverlb.SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort) +} + +func supportedAPIServerListenerPortsString(apiServerPort *int32) string { + return apiserverlb.SupportedAPIServerListenerPortsString(apiServerPort, supportedSecondaryListenerPort) +} diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 2f3234da0..cf5ec2af3 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -1061,15 +1061,7 @@ type HealthChecker struct { // 1. If `backendSets` is configured, it is used as-is. // 2. Otherwise, a single backend set named APIServerLBBackendSetName is synthesized from the legacy `backendSetDetails`. func (in *NLBSpec) CanonicalBackendSets() []NLBBackendSet { - if len(in.BackendSets) > 0 { - return append([]NLBBackendSet(nil), in.BackendSets...) - } - return []NLBBackendSet{ - { - Name: APIServerLBBackendSetName, - BackendSetDetails: in.BackendSetDetails, - }, - } + return canonicalAPIServerBackendSets(in.BackendSets, in.BackendSetDetails) } // NetworkSpec specifies what the OCI networking resources should look like. diff --git a/api/v1beta1/validator.go b/api/v1beta1/validator.go index 2806e5ebd..8076f6be8 100644 --- a/api/v1beta1/validator.go +++ b/api/v1beta1/validator.go @@ -23,7 +23,6 @@ import ( "fmt" "net" "regexp" - "strings" "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" "k8s.io/apimachinery/pkg/util/validation/field" @@ -97,134 +96,7 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network } func validateAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { - var allErrs field.ErrorList - - legacyConfigured := spec.BackendSetDetails != (BackendSetDetails{}) - if len(spec.BackendSets) > 0 && legacyConfigured { - allErrs = append(allErrs, field.Forbidden( - fldPath.Child("backendSetDetails"), - "cannot be set when backendSets is configured; move legacy backendSetDetails into backendSets", - )) - } - - nameRegex := regexp.MustCompile(apiServerBackendSetNameRegex) - seen := map[string]int{} - for i, backendSet := range spec.BackendSets { - namePath := fldPath.Child("backendSets").Index(i).Child("name") - name := strings.TrimSpace(backendSet.Name) - if name == "" { - allErrs = append(allErrs, field.Required(namePath, "must not be empty")) - continue - } - if !nameRegex.MatchString(name) { - allErrs = append(allErrs, field.Invalid( - namePath, - backendSet.Name, - "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$ (1-32 chars; alphanumeric, '-', '_')", - )) - } - if firstIdx, ok := seen[name]; ok { - allErrs = append(allErrs, field.Invalid( - namePath, - backendSet.Name, - fmt.Sprintf("duplicate backend set name, already used at backendSets[%d].name", firstIdx), - )) - continue - } - seen[name] = i - } - - usedPorts := map[int32]int{} - omittedListenerPortIndex := -1 - for i, backendSet := range spec.BackendSets { - portPath := fldPath.Child("backendSets").Index(i).Child("listenerPort") - if backendSet.ListenerPort != nil { - port := *backendSet.ListenerPort - if port < 1 || port > 65535 { - allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) - continue - } - if !isSupportedAPIServerListenerPort(port, apiServerPort) { - allErrs = append(allErrs, field.Invalid( - portPath, - port, - fmt.Sprintf("must be one of %s to avoid exposing arbitrary control-plane ports", supportedAPIServerListenerPortsString(apiServerPort)), - )) - continue - } - } - - port, known := effectiveAPIServerListenerPort(backendSet.ListenerPort, apiServerPort) - if !known { - if omittedListenerPortIndex >= 0 { - allErrs = append(allErrs, field.Invalid( - portPath, - backendSet.ListenerPort, - fmt.Sprintf("multiple backend sets omit listenerPort, already omitted at backendSets[%d].listenerPort", omittedListenerPortIndex), - )) - continue - } - omittedListenerPortIndex = i - continue - } - - if firstIdx, ok := usedPorts[port]; ok { - allErrs = append(allErrs, field.Invalid( - portPath, - backendSet.ListenerPort, - fmt.Sprintf("duplicate effective listenerPort %d, already used at backendSets[%d].listenerPort", port, firstIdx), - )) - continue - } - usedPorts[port] = i - } - - return allErrs -} - -func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { - if listenerPort != nil { - return *listenerPort, true - } - if apiServerPort == nil { - return 0, false - } - - return *apiServerPort, true -} - -func isSupportedAPIServerListenerPort(port int32, apiServerPort *int32) bool { - for _, allowedPort := range supportedAPIServerListenerPorts(apiServerPort) { - if port == allowedPort { - return true - } - } - return false -} - -func supportedAPIServerListenerPorts(apiServerPort *int32) []int32 { - ports := []int32{ociutil.DefaultAPIServerPort} - if apiServerPort != nil && *apiServerPort != ociutil.DefaultAPIServerPort { - ports = append(ports, *apiServerPort) - } - if supportedSecondaryListenerPort != ociutil.DefaultAPIServerPort { - ports = append(ports, supportedSecondaryListenerPort) - } - return ports -} - -func supportedAPIServerListenerPortsString(apiServerPort *int32) string { - values := supportedAPIServerListenerPorts(apiServerPort) - parts := make([]string, 0, len(values)) - seen := map[int32]struct{}{} - for _, value := range values { - if _, ok := seen[value]; ok { - continue - } - seen[value] = struct{}{} - parts = append(parts, fmt.Sprintf("%d", value)) - } - return strings.Join(parts, ", ") + return validateSharedAPIServerLBBackendSets(spec, apiServerPort, fldPath) } // validateVCNCIDR validates the CIDR of a VNC. diff --git a/api/v1beta2/backendsets_shared.go b/api/v1beta2/backendsets_shared.go new file mode 100644 index 000000000..abd5bf4bb --- /dev/null +++ b/api/v1beta2/backendsets_shared.go @@ -0,0 +1,42 @@ +package v1beta2 + +import ( + apiserverlb "github.com/oracle/cluster-api-provider-oci/api/internal/apiserverlb" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func canonicalAPIServerBackendSets(configured []NLBBackendSet, legacy BackendSetDetails) []NLBBackendSet { + return apiserverlb.CanonicalBackendSets(configured, func() NLBBackendSet { + return NLBBackendSet{ + Name: APIServerLBBackendSetName, + BackendSetDetails: legacy, + } + }) +} + +func validateSharedAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { + backendSets := make([]apiserverlb.BackendSet, 0, len(spec.BackendSets)) + for _, backendSet := range spec.BackendSets { + backendSets = append(backendSets, apiserverlb.BackendSet{ + Name: backendSet.Name, + ListenerPort: backendSet.ListenerPort, + }) + } + return apiserverlb.ValidateBackendSets(backendSets, spec.BackendSetDetails != (BackendSetDetails{}), apiServerPort, supportedSecondaryListenerPort, fldPath) +} + +func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { + return apiserverlb.EffectiveAPIServerListenerPort(listenerPort, apiServerPort) +} + +func isSupportedAPIServerListenerPort(port int32, apiServerPort *int32) bool { + return apiserverlb.IsSupportedAPIServerListenerPort(port, apiServerPort, supportedSecondaryListenerPort) +} + +func supportedAPIServerListenerPorts(apiServerPort *int32) []int32 { + return apiserverlb.SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort) +} + +func supportedAPIServerListenerPortsString(apiServerPort *int32) string { + return apiserverlb.SupportedAPIServerListenerPortsString(apiServerPort, supportedSecondaryListenerPort) +} diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index acc049907..bf8f405db 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -1071,15 +1071,7 @@ type HealthChecker struct { // 1. If `backendSets` is configured, it is used as-is. // 2. Otherwise, a single backend set named APIServerLBBackendSetName is synthesized from the legacy `backendSetDetails`. func (in *NLBSpec) CanonicalBackendSets() []NLBBackendSet { - if len(in.BackendSets) > 0 { - return append([]NLBBackendSet(nil), in.BackendSets...) - } - return []NLBBackendSet{ - { - Name: APIServerLBBackendSetName, - BackendSetDetails: in.BackendSetDetails, - }, - } + return canonicalAPIServerBackendSets(in.BackendSets, in.BackendSetDetails) } // NetworkSpec specifies what the OCI networking resources should look like. diff --git a/api/v1beta2/validator.go b/api/v1beta2/validator.go index 059f63254..f48d4c256 100644 --- a/api/v1beta2/validator.go +++ b/api/v1beta2/validator.go @@ -23,7 +23,6 @@ import ( "fmt" "net" "regexp" - "strings" "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" "k8s.io/apimachinery/pkg/util/validation/field" @@ -138,134 +137,7 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network } func validateAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { - var allErrs field.ErrorList - - legacyConfigured := spec.BackendSetDetails != (BackendSetDetails{}) - if len(spec.BackendSets) > 0 && legacyConfigured { - allErrs = append(allErrs, field.Forbidden( - fldPath.Child("backendSetDetails"), - "cannot be set when backendSets is configured; move legacy backendSetDetails into backendSets", - )) - } - - nameRegex := regexp.MustCompile(apiServerBackendSetNameRegex) - seen := map[string]int{} - for i, backendSet := range spec.BackendSets { - namePath := fldPath.Child("backendSets").Index(i).Child("name") - name := strings.TrimSpace(backendSet.Name) - if name == "" { - allErrs = append(allErrs, field.Required(namePath, "must not be empty")) - continue - } - if !nameRegex.MatchString(name) { - allErrs = append(allErrs, field.Invalid( - namePath, - backendSet.Name, - "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$ (1-32 chars; alphanumeric, '-', '_')", - )) - } - if firstIdx, ok := seen[name]; ok { - allErrs = append(allErrs, field.Invalid( - namePath, - backendSet.Name, - fmt.Sprintf("duplicate backend set name, already used at backendSets[%d].name", firstIdx), - )) - continue - } - seen[name] = i - } - - usedPorts := map[int32]int{} - omittedListenerPortIndex := -1 - for i, backendSet := range spec.BackendSets { - portPath := fldPath.Child("backendSets").Index(i).Child("listenerPort") - if backendSet.ListenerPort != nil { - port := *backendSet.ListenerPort - if port < 1 || port > 65535 { - allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) - continue - } - if !isSupportedAPIServerListenerPort(port, apiServerPort) { - allErrs = append(allErrs, field.Invalid( - portPath, - port, - fmt.Sprintf("must be one of %s to avoid exposing arbitrary control-plane ports", supportedAPIServerListenerPortsString(apiServerPort)), - )) - continue - } - } - - port, known := effectiveAPIServerListenerPort(backendSet.ListenerPort, apiServerPort) - if !known { - if omittedListenerPortIndex >= 0 { - allErrs = append(allErrs, field.Invalid( - portPath, - backendSet.ListenerPort, - fmt.Sprintf("multiple backend sets omit listenerPort, already omitted at backendSets[%d].listenerPort", omittedListenerPortIndex), - )) - continue - } - omittedListenerPortIndex = i - continue - } - - if firstIdx, ok := usedPorts[port]; ok { - allErrs = append(allErrs, field.Invalid( - portPath, - backendSet.ListenerPort, - fmt.Sprintf("duplicate effective listenerPort %d, already used at backendSets[%d].listenerPort", port, firstIdx), - )) - continue - } - usedPorts[port] = i - } - - return allErrs -} - -func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { - if listenerPort != nil { - return *listenerPort, true - } - if apiServerPort == nil { - return 0, false - } - - return *apiServerPort, true -} - -func isSupportedAPIServerListenerPort(port int32, apiServerPort *int32) bool { - for _, allowedPort := range supportedAPIServerListenerPorts(apiServerPort) { - if port == allowedPort { - return true - } - } - return false -} - -func supportedAPIServerListenerPorts(apiServerPort *int32) []int32 { - ports := []int32{ociutil.DefaultAPIServerPort} - if apiServerPort != nil && *apiServerPort != ociutil.DefaultAPIServerPort { - ports = append(ports, *apiServerPort) - } - if supportedSecondaryListenerPort != ociutil.DefaultAPIServerPort { - ports = append(ports, supportedSecondaryListenerPort) - } - return ports -} - -func supportedAPIServerListenerPortsString(apiServerPort *int32) string { - values := supportedAPIServerListenerPorts(apiServerPort) - parts := make([]string, 0, len(values)) - seen := map[int32]struct{}{} - for _, value := range values { - if _, ok := seen[value]; ok { - continue - } - seen[value] = struct{}{} - parts = append(parts, fmt.Sprintf("%d", value)) - } - return strings.Join(parts, ", ") + return validateSharedAPIServerLBBackendSets(spec, apiServerPort, fldPath) } // validateVCNCIDR validates the CIDR of a VNC. From a0b13f67caa9f4f1dab8c76448a5dd3d3471d65d Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 19:37:10 -0400 Subject: [PATCH 26/31] Refactor API server LB resource reconciliation --- cloud/scope/load_balancer_reconciler.go | 146 ++++++++------- .../scope/named_resource_reconcile_helpers.go | 67 +++++++ .../scope/network_load_balancer_reconciler.go | 170 +++++++++--------- 3 files changed, 229 insertions(+), 154 deletions(-) create mode 100644 cloud/scope/named_resource_reconcile_helpers.go diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index 1752e56df..f8645ce4d 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -156,9 +156,10 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu } desiredListeners, desiredBackendSets := s.buildDesiredLBListenersAndBackendSets(lb) - for name, desired := range desiredBackendSets { - actual, exists := actualResp.LoadBalancer.BackendSets[name] - if !exists { + if err := reconcileNamedResources( + actualResp.LoadBalancer.BackendSets, + desiredBackendSets, + func(name string, desired loadbalancer.BackendSetDetails) error { resp, err := s.LoadBalancerClient.CreateBackendSet(ctx, loadbalancer.CreateBackendSetRequest{ LoadBalancerId: lbID, CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ @@ -170,19 +171,17 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu if err != nil { if isAlreadyExistsOCIError(err) { s.Logger.Info("LB backend set already exists during reconcile; continuing", "backendSet", name) - continue + return nil } return errors.Wrapf(err, "failed to create lb backend set %q", name) } if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting create backend set %q", name) } - continue - } - if !ptr.StringEquals(actual.Policy, ptr.ToString(desired.Policy)) || - actual.HealthChecker == nil || desired.HealthChecker == nil || - !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || - !ptr.StringEquals(actual.HealthChecker.Protocol, ptr.ToString(desired.HealthChecker.Protocol)) { + return nil + }, + lbBackendSetNeedsUpdate, + func(name string, desired loadbalancer.BackendSetDetails) error { resp, err := s.LoadBalancerClient.UpdateBackendSet(ctx, loadbalancer.UpdateBackendSetRequest{ LoadBalancerId: lbID, BackendSetName: common.String(name), @@ -198,12 +197,16 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting update backend set %q", name) } - } + return nil + }, + ); err != nil { + return err } - for name, desired := range desiredListeners { - actual, exists := actualResp.LoadBalancer.Listeners[name] - if !exists { + if err := reconcileNamedResources( + actualResp.LoadBalancer.Listeners, + desiredListeners, + func(name string, desired loadbalancer.ListenerDetails) error { resp, err := s.LoadBalancerClient.CreateListener(ctx, loadbalancer.CreateListenerRequest{ LoadBalancerId: lbID, CreateListenerDetails: loadbalancer.CreateListenerDetails{ @@ -216,18 +219,17 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu if err != nil { if isAlreadyExistsOCIError(err) { s.Logger.Info("LB listener already exists during reconcile; continuing", "listener", name) - continue + return nil } return errors.Wrapf(err, "failed to create lb listener %q", name) } if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting create listener %q", name) } - continue - } - if !intPtrEqual(actual.Port, desired.Port) || - !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || - !ptr.StringEquals(actual.Protocol, ptr.ToString(desired.Protocol)) { + return nil + }, + lbListenerNeedsUpdate, + func(name string, desired loadbalancer.ListenerDetails) error { resp, err := s.LoadBalancerClient.UpdateListener(ctx, loadbalancer.UpdateListenerRequest{ LoadBalancerId: lbID, ListenerName: common.String(name), @@ -243,24 +245,31 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting update listener %q", name) } - } + return nil + }, + ); err != nil { + return err } - staleListenerDeleted := false - for name := range actualResp.LoadBalancer.Listeners { - if _, keep := desiredListeners[name]; keep { - continue - } - resp, err := s.LoadBalancerClient.DeleteListener(ctx, loadbalancer.DeleteListenerRequest{ - LoadBalancerId: lbID, - ListenerName: common.String(name), - }) - if err != nil { - return errors.Wrapf(err, "failed to delete stale lb listener %q", name) - } - if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { - return errors.Wrapf(err, "failed awaiting delete listener %q", name) - } - staleListenerDeleted = true + staleListenerDeleted, err := deleteStaleNamedResources( + actualResp.LoadBalancer.Listeners, + desiredNameSet(desiredListeners), + func(loadbalancer.Listener) bool { return true }, + func(name string, _ loadbalancer.Listener) error { + resp, err := s.LoadBalancerClient.DeleteListener(ctx, loadbalancer.DeleteListenerRequest{ + LoadBalancerId: lbID, + ListenerName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale lb listener %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete listener %q", name) + } + return nil + }, + ) + if err != nil { + return err } if staleListenerDeleted { actualResp, err = s.LoadBalancerClient.GetLoadBalancer(ctx, loadbalancer.GetLoadBalancerRequest{ @@ -270,23 +279,25 @@ func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructu return errors.Wrap(err, "failed to refresh lb after stale listener reconciliation") } } - for name, actual := range actualResp.LoadBalancer.BackendSets { - if _, keep := desiredBackendSets[name]; keep { - continue - } - if len(actual.Backends) > 0 { - continue - } - resp, err := s.LoadBalancerClient.DeleteBackendSet(ctx, loadbalancer.DeleteBackendSetRequest{ - LoadBalancerId: lbID, - BackendSetName: common.String(name), - }) - if err != nil { - return errors.Wrapf(err, "failed to delete stale lb backend set %q", name) - } - if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { - return errors.Wrapf(err, "failed awaiting delete backend set %q", name) - } + if _, err := deleteStaleNamedResources( + actualResp.LoadBalancer.BackendSets, + desiredNameSet(desiredBackendSets), + func(actual loadbalancer.BackendSet) bool { return len(actual.Backends) == 0 }, + func(name string, _ loadbalancer.BackendSet) error { + resp, err := s.LoadBalancerClient.DeleteBackendSet(ctx, loadbalancer.DeleteBackendSetRequest{ + LoadBalancerId: lbID, + BackendSetName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale lb backend set %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete backend set %q", name) + } + return nil + }, + ); err != nil { + return err } return nil } @@ -365,10 +376,7 @@ func (s *ClusterScope) CreateLB(ctx context.Context, lb infrastructurev1beta2.Lo } func (s *ClusterScope) buildDesiredLBListenersAndBackendSets(lb infrastructurev1beta2.LoadBalancer) (map[string]loadbalancer.ListenerDetails, map[string]loadbalancer.BackendSetDetails) { - canonicalBackendSets := lb.NLBSpec.CanonicalBackendSets() - if len(canonicalBackendSets) == 0 { - canonicalBackendSets = []infrastructurev1beta2.NLBBackendSet{{Name: APIServerLBBackendSetName}} - } + canonicalBackendSets := desiredAPIServerBackendSets(lb) backendSetDetails := make(map[string]loadbalancer.BackendSetDetails, len(canonicalBackendSets)) for _, backendSet := range canonicalBackendSets { @@ -431,9 +439,7 @@ func (s *ClusterScope) IsLBEqual(actual *loadbalancer.LoadBalancer, desired infr if !exists { return false } - if !intPtrEqual(actualListener.Port, desiredListener.Port) || - !ptr.StringEquals(actualListener.DefaultBackendSetName, ptr.ToString(desiredListener.DefaultBackendSetName)) || - !ptr.StringEquals(actualListener.Protocol, ptr.ToString(desiredListener.Protocol)) { + if lbListenerNeedsUpdate(actualListener, desiredListener) { return false } } @@ -446,10 +452,7 @@ func (s *ClusterScope) IsLBEqual(actual *loadbalancer.LoadBalancer, desired infr if !exists { return false } - if !ptr.StringEquals(actualBackendSet.Policy, ptr.ToString(desiredBackendSet.Policy)) || - actualBackendSet.HealthChecker == nil || desiredBackendSet.HealthChecker == nil || - !intPtrEqual(actualBackendSet.HealthChecker.Port, desiredBackendSet.HealthChecker.Port) || - !ptr.StringEquals(actualBackendSet.HealthChecker.Protocol, ptr.ToString(desiredBackendSet.HealthChecker.Protocol)) { + if lbBackendSetNeedsUpdate(actualBackendSet, desiredBackendSet) { return false } } @@ -457,6 +460,19 @@ func (s *ClusterScope) IsLBEqual(actual *loadbalancer.LoadBalancer, desired infr return true } +func lbListenerNeedsUpdate(actual loadbalancer.Listener, desired loadbalancer.ListenerDetails) bool { + return !intPtrEqual(actual.Port, desired.Port) || + !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || + !ptr.StringEquals(actual.Protocol, ptr.ToString(desired.Protocol)) +} + +func lbBackendSetNeedsUpdate(actual loadbalancer.BackendSet, desired loadbalancer.BackendSetDetails) bool { + return !ptr.StringEquals(actual.Policy, ptr.ToString(desired.Policy)) || + actual.HealthChecker == nil || desired.HealthChecker == nil || + !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || + !ptr.StringEquals(actual.HealthChecker.Protocol, ptr.ToString(desired.HealthChecker.Protocol)) +} + // GetLoadBalancers retrieves the Cluster's loadbalancer.LoadBalancer using the one of the following methods // // 1. the OCICluster's spec LoadBalancerId diff --git a/cloud/scope/named_resource_reconcile_helpers.go b/cloud/scope/named_resource_reconcile_helpers.go new file mode 100644 index 000000000..dbd5a3d14 --- /dev/null +++ b/cloud/scope/named_resource_reconcile_helpers.go @@ -0,0 +1,67 @@ +package scope + +import infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" + +func desiredAPIServerBackendSets(lb infrastructurev1beta2.LoadBalancer) []infrastructurev1beta2.NLBBackendSet { + canonicalBackendSets := lb.NLBSpec.CanonicalBackendSets() + if len(canonicalBackendSets) > 0 { + return canonicalBackendSets + } + + return []infrastructurev1beta2.NLBBackendSet{{Name: APIServerLBBackendSetName}} +} + +func reconcileNamedResources[Actual any, Desired any]( + actual map[string]Actual, + desired map[string]Desired, + create func(name string, desired Desired) error, + needsUpdate func(actual Actual, desired Desired) bool, + update func(name string, desired Desired) error, +) error { + for name, desiredResource := range desired { + actualResource, exists := actual[name] + if !exists { + if err := create(name, desiredResource); err != nil { + return err + } + continue + } + if !needsUpdate(actualResource, desiredResource) { + continue + } + if err := update(name, desiredResource); err != nil { + return err + } + } + return nil +} + +func deleteStaleNamedResources[Actual any, Desired any]( + actual map[string]Actual, + desired map[string]Desired, + shouldDelete func(actual Actual) bool, + deleteFn func(name string, actual Actual) error, +) (bool, error) { + deleted := false + for name, actualResource := range actual { + if _, keep := desired[name]; keep { + continue + } + if shouldDelete != nil && !shouldDelete(actualResource) { + continue + } + if err := deleteFn(name, actualResource); err != nil { + return deleted, err + } + deleted = true + } + return deleted, nil +} + +func desiredNameSet[T any](resources map[string]T) map[string]struct{} { + names := make(map[string]struct{}, len(resources)) + for name := range resources { + names[name] = struct{}{} + } + return names +} diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index 3781ff94a..d53a56775 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -150,9 +150,10 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc } desiredListeners, desiredBackendSets := s.buildDesiredNLBListenersAndBackendSets(nlb) - for name, desired := range desiredListeners { - actual, exists := actualResp.NetworkLoadBalancer.Listeners[name] - if !exists { + if err := reconcileNamedResources( + actualResp.NetworkLoadBalancer.Listeners, + desiredListeners, + func(name string, desired networkloadbalancer.ListenerDetails) error { resp, err := s.NetworkLoadBalancerClient.CreateListener(ctx, networkloadbalancer.CreateListenerRequest{ NetworkLoadBalancerId: nlbID, CreateListenerDetails: networkloadbalancer.CreateListenerDetails{ @@ -165,18 +166,17 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc if err != nil { if isAlreadyExistsOCIError(err) { s.Logger.Info("NLB listener already exists during reconcile; continuing", "listener", name) - continue + return nil } return errors.Wrapf(err, "failed to create nlb listener %q", name) } if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting create listener %q", name) } - continue - } - if !intPtrEqual(actual.Port, desired.Port) || - !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || - actual.Protocol != desired.Protocol { + return nil + }, + nlbListenerNeedsUpdate, + func(name string, desired networkloadbalancer.ListenerDetails) error { resp, err := s.NetworkLoadBalancerClient.UpdateListener(ctx, networkloadbalancer.UpdateListenerRequest{ NetworkLoadBalancerId: nlbID, ListenerName: common.String(name), @@ -192,24 +192,31 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting update listener %q", name) } - } + return nil + }, + ); err != nil { + return err } - staleListenerDeleted := false - for name := range actualResp.NetworkLoadBalancer.Listeners { - if _, keep := desiredListeners[name]; keep { - continue - } - resp, err := s.NetworkLoadBalancerClient.DeleteListener(ctx, networkloadbalancer.DeleteListenerRequest{ - NetworkLoadBalancerId: nlbID, - ListenerName: common.String(name), - }) - if err != nil { - return errors.Wrapf(err, "failed to delete stale nlb listener %q", name) - } - if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { - return errors.Wrapf(err, "failed awaiting delete listener %q", name) - } - staleListenerDeleted = true + staleListenerDeleted, err := deleteStaleNamedResources( + actualResp.NetworkLoadBalancer.Listeners, + desiredNameSet(desiredListeners), + func(networkloadbalancer.Listener) bool { return true }, + func(name string, _ networkloadbalancer.Listener) error { + resp, err := s.NetworkLoadBalancerClient.DeleteListener(ctx, networkloadbalancer.DeleteListenerRequest{ + NetworkLoadBalancerId: nlbID, + ListenerName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale nlb listener %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete listener %q", name) + } + return nil + }, + ) + if err != nil { + return err } if staleListenerDeleted { actualResp, err = s.NetworkLoadBalancerClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ @@ -220,9 +227,10 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc } } - for name, desired := range desiredBackendSets { - actual, exists := actualResp.NetworkLoadBalancer.BackendSets[name] - if !exists { + if err := reconcileNamedResources( + actualResp.NetworkLoadBalancer.BackendSets, + desiredBackendSets, + func(name string, desired networkloadbalancer.BackendSetDetails) error { resp, err := s.NetworkLoadBalancerClient.CreateBackendSet(ctx, networkloadbalancer.CreateBackendSetRequest{ NetworkLoadBalancerId: nlbID, CreateBackendSetDetails: networkloadbalancer.CreateBackendSetDetails{ @@ -237,17 +245,17 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc if err != nil { if isAlreadyExistsOCIError(err) { s.Logger.Info("NLB backend set already exists during reconcile; continuing", "backendSet", name) - continue + return nil } return errors.Wrapf(err, "failed to create nlb backend set %q", name) } if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting create backend set %q", name) } - continue - } - if actual.HealthChecker == nil || desired.HealthChecker == nil { - // Reconcile nullable health checker by updating if either side is nil. + return nil + }, + nlbBackendSetNeedsUpdate, + func(name string, desired networkloadbalancer.BackendSetDetails) error { resp, err := s.NetworkLoadBalancerClient.UpdateBackendSet(ctx, networkloadbalancer.UpdateBackendSetRequest{ NetworkLoadBalancerId: nlbID, BackendSetName: common.String(name), @@ -265,51 +273,30 @@ func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastruc if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { return errors.Wrapf(err, "failed awaiting update backend set %q", name) } - continue - } - if actual.Policy != desired.Policy || - !boolPtrEqual(actual.IsPreserveSource, desired.IsPreserveSource) || - !boolPtrEqual(actual.IsFailOpen, desired.IsFailOpen) || - !boolPtrEqual(actual.IsInstantFailoverEnabled, desired.IsInstantFailoverEnabled) || - !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || - actual.HealthChecker.Protocol != desired.HealthChecker.Protocol || - !ptr.StringEquals(actual.HealthChecker.UrlPath, ptr.ToString(desired.HealthChecker.UrlPath)) { - resp, err := s.NetworkLoadBalancerClient.UpdateBackendSet(ctx, networkloadbalancer.UpdateBackendSetRequest{ + return nil + }, + ); err != nil { + return err + } + if _, err := deleteStaleNamedResources( + actualResp.NetworkLoadBalancer.BackendSets, + desiredNameSet(desiredBackendSets), + func(actual networkloadbalancer.BackendSet) bool { return len(actual.Backends) == 0 }, + func(name string, _ networkloadbalancer.BackendSet) error { + resp, err := s.NetworkLoadBalancerClient.DeleteBackendSet(ctx, networkloadbalancer.DeleteBackendSetRequest{ NetworkLoadBalancerId: nlbID, BackendSetName: common.String(name), - UpdateBackendSetDetails: networkloadbalancer.UpdateBackendSetDetails{ - Policy: common.String(string(desired.Policy)), - HealthChecker: nlbHealthCheckerToDetails(desired.HealthChecker), - IsPreserveSource: desired.IsPreserveSource, - IsFailOpen: desired.IsFailOpen, - IsInstantFailoverEnabled: desired.IsInstantFailoverEnabled, - }, }) if err != nil { - return errors.Wrapf(err, "failed to update nlb backend set %q", name) + return errors.Wrapf(err, "failed to delete stale nlb backend set %q", name) } if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { - return errors.Wrapf(err, "failed awaiting update backend set %q", name) + return errors.Wrapf(err, "failed awaiting delete backend set %q", name) } - } - } - for name, actual := range actualResp.NetworkLoadBalancer.BackendSets { - if _, keep := desiredBackendSets[name]; keep { - continue - } - if len(actual.Backends) > 0 { - continue - } - resp, err := s.NetworkLoadBalancerClient.DeleteBackendSet(ctx, networkloadbalancer.DeleteBackendSetRequest{ - NetworkLoadBalancerId: nlbID, - BackendSetName: common.String(name), - }) - if err != nil { - return errors.Wrapf(err, "failed to delete stale nlb backend set %q", name) - } - if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { - return errors.Wrapf(err, "failed awaiting delete backend set %q", name) - } + return nil + }, + ); err != nil { + return err } return nil } @@ -397,10 +384,7 @@ func (s *ClusterScope) CreateNLB(ctx context.Context, lb infrastructurev1beta2.L } func (s *ClusterScope) buildDesiredNLBListenersAndBackendSets(lb infrastructurev1beta2.LoadBalancer) (map[string]networkloadbalancer.ListenerDetails, map[string]networkloadbalancer.BackendSetDetails) { - canonicalBackendSets := lb.NLBSpec.CanonicalBackendSets() - if len(canonicalBackendSets) == 0 { - canonicalBackendSets = []infrastructurev1beta2.NLBBackendSet{{Name: APIServerLBBackendSetName}} - } + canonicalBackendSets := desiredAPIServerBackendSets(lb) backendSetDetails := make(map[string]networkloadbalancer.BackendSetDetails, len(canonicalBackendSets)) for _, backendSet := range canonicalBackendSets { @@ -509,9 +493,7 @@ func (s *ClusterScope) IsNLBEqual(actual *networkloadbalancer.NetworkLoadBalance if !exists { return false } - if !intPtrEqual(actualListener.Port, desiredListener.Port) || - !ptr.StringEquals(actualListener.DefaultBackendSetName, ptr.ToString(desiredListener.DefaultBackendSetName)) || - actualListener.Protocol != desiredListener.Protocol { + if nlbListenerNeedsUpdate(actualListener, desiredListener) { return false } } @@ -524,16 +506,7 @@ func (s *ClusterScope) IsNLBEqual(actual *networkloadbalancer.NetworkLoadBalance if !exists { return false } - if actualBackendSet.HealthChecker == nil || desiredBackendSet.HealthChecker == nil { - return false - } - if actualBackendSet.Policy != desiredBackendSet.Policy || - !boolPtrEqual(actualBackendSet.IsPreserveSource, desiredBackendSet.IsPreserveSource) || - !boolPtrEqual(actualBackendSet.IsFailOpen, desiredBackendSet.IsFailOpen) || - !boolPtrEqual(actualBackendSet.IsInstantFailoverEnabled, desiredBackendSet.IsInstantFailoverEnabled) || - !intPtrEqual(actualBackendSet.HealthChecker.Port, desiredBackendSet.HealthChecker.Port) || - actualBackendSet.HealthChecker.Protocol != desiredBackendSet.HealthChecker.Protocol || - !ptr.StringEquals(actualBackendSet.HealthChecker.UrlPath, ptr.ToString(desiredBackendSet.HealthChecker.UrlPath)) { + if nlbBackendSetNeedsUpdate(actualBackendSet, desiredBackendSet) { return false } } @@ -541,6 +514,25 @@ func (s *ClusterScope) IsNLBEqual(actual *networkloadbalancer.NetworkLoadBalance return true } +func nlbListenerNeedsUpdate(actual networkloadbalancer.Listener, desired networkloadbalancer.ListenerDetails) bool { + return !intPtrEqual(actual.Port, desired.Port) || + !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || + actual.Protocol != desired.Protocol +} + +func nlbBackendSetNeedsUpdate(actual networkloadbalancer.BackendSet, desired networkloadbalancer.BackendSetDetails) bool { + if actual.HealthChecker == nil || desired.HealthChecker == nil { + return true + } + return actual.Policy != desired.Policy || + !boolPtrEqual(actual.IsPreserveSource, desired.IsPreserveSource) || + !boolPtrEqual(actual.IsFailOpen, desired.IsFailOpen) || + !boolPtrEqual(actual.IsInstantFailoverEnabled, desired.IsInstantFailoverEnabled) || + !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || + actual.HealthChecker.Protocol != desired.HealthChecker.Protocol || + !ptr.StringEquals(actual.HealthChecker.UrlPath, ptr.ToString(desired.HealthChecker.UrlPath)) +} + // GetNetworkLoadBalancers retrieves the Cluster's networkloadbalancer.NetworkLoadBalancer using the one of the following methods // // 1. the OCICluster's spec LoadBalancerId From eec40f150f682caf3c2b4fc81d5266e83913c9cb Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 19:41:00 -0400 Subject: [PATCH 27/31] Clarify shared API server LB naming --- api/v1beta1/types.go | 28 +++++++++++++++++-- api/v1beta2/types.go | 28 +++++++++++++++++-- cloud/scope/apiserver_lb_backendsets_test.go | 2 +- cloud/scope/load_balancer_reconciler.go | 16 +++++------ cloud/scope/machine.go | 2 +- .../scope/named_resource_reconcile_helpers.go | 2 +- .../scope/network_load_balancer_reconciler.go | 19 ++++++------- test/e2e/cluster_test.go | 2 +- 8 files changed, 70 insertions(+), 29 deletions(-) diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index cf5ec2af3..499dc899c 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -988,7 +988,27 @@ type LoadBalancer struct { NLBSpec NLBSpec `json:"nlbSpec,omitempty"` } -// NLBSpec specifies the NLB spec. +// NewAPIServerLoadBalancer returns a LoadBalancer configured with the shared API server settings. +// The underlying `nlbSpec` field name is historical and retained for API compatibility. +func NewAPIServerLoadBalancer(name string, spec NLBSpec) LoadBalancer { + return LoadBalancer{ + Name: name, + NLBSpec: spec, + } +} + +// APIServerLoadBalancerSpec returns the shared API server load balancer settings. +func (in *LoadBalancer) APIServerLoadBalancerSpec() *NLBSpec { + return &in.NLBSpec +} + +// CanonicalAPIServerBackendSets returns the canonical API server backend set list for this load balancer. +func (in *LoadBalancer) CanonicalAPIServerBackendSets() []NLBBackendSet { + return in.APIServerLoadBalancerSpec().CanonicalBackendSets() +} + +// NLBSpec specifies the shared API server load balancer settings. +// The type name is historical and applies to both `loadBalancerType: nlb` and `loadBalancerType: lb`. type NLBSpec struct { // BackendSets specifies the canonical list of API server backend sets. // When set, this field takes precedence over the legacy `backendSetDetails` field. @@ -1009,7 +1029,8 @@ type NLBSpec struct { ReservedIpIds []string `json:"reservedIpIds,omitempty"` } -// NLBBackendSet specifies the configuration for a named network load balancer backend set. +// NLBBackendSet specifies the configuration for a named API server backend set. +// The type name is historical and applies to both load balancer implementations. type NLBBackendSet struct { // Name is the API server backend set identifier. Name string `json:"name"` @@ -1025,7 +1046,8 @@ type NLBBackendSet struct { BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` } -// BackendSetDetails specifies the configuration of a network load balancer backend set. +// BackendSetDetails specifies the configuration of an API server backend set. +// The fields cover shared behavior used by both load balancer implementations. type BackendSetDetails struct { // If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. // Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index bf8f405db..54f83cc5f 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -997,7 +997,27 @@ type LoadBalancer struct { NLBSpec NLBSpec `json:"nlbSpec,omitempty"` } -// NLBSpec specifies the NLB spec. +// NewAPIServerLoadBalancer returns a LoadBalancer configured with the shared API server settings. +// The underlying `nlbSpec` field name is historical and retained for API compatibility. +func NewAPIServerLoadBalancer(name string, spec NLBSpec) LoadBalancer { + return LoadBalancer{ + Name: name, + NLBSpec: spec, + } +} + +// APIServerLoadBalancerSpec returns the shared API server load balancer settings. +func (in *LoadBalancer) APIServerLoadBalancerSpec() *NLBSpec { + return &in.NLBSpec +} + +// CanonicalAPIServerBackendSets returns the canonical API server backend set list for this load balancer. +func (in *LoadBalancer) CanonicalAPIServerBackendSets() []NLBBackendSet { + return in.APIServerLoadBalancerSpec().CanonicalBackendSets() +} + +// NLBSpec specifies the shared API server load balancer settings. +// The type name is historical and applies to both `loadBalancerType: nlb` and `loadBalancerType: lb`. type NLBSpec struct { // BackendSets specifies the canonical list of API server backend sets. // When set, this field takes precedence over the legacy `backendSetDetails` field. @@ -1018,7 +1038,8 @@ type NLBSpec struct { ReservedIpIds []string `json:"reservedIpIds,omitempty"` } -// NLBBackendSet specifies the configuration for a named network load balancer backend set. +// NLBBackendSet specifies the configuration for a named API server backend set. +// The type name is historical and applies to both load balancer implementations. type NLBBackendSet struct { // Name is the API server backend set identifier. Name string `json:"name"` @@ -1034,7 +1055,8 @@ type NLBBackendSet struct { BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` } -// BackendSetDetails specifies the configuration of a network load balancer backend set. +// BackendSetDetails specifies the configuration of an API server backend set. +// The fields cover shared behavior used by both load balancer implementations. type BackendSetDetails struct { // If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. // Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. diff --git a/cloud/scope/apiserver_lb_backendsets_test.go b/cloud/scope/apiserver_lb_backendsets_test.go index 76ce5842a..24a035308 100644 --- a/cloud/scope/apiserver_lb_backendsets_test.go +++ b/cloud/scope/apiserver_lb_backendsets_test.go @@ -96,7 +96,7 @@ func TestLBSpecPreservesBackendSets(t *testing.T) { }, } - got := scope.LBSpec() + got := scope.DesiredAPIServerLoadBalancer() if len(got.NLBSpec.BackendSets) != 2 { t.Fatalf("expected backend sets to be preserved, got %#v", got.NLBSpec.BackendSets) } diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index f8645ce4d..76a6490fb 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -31,7 +31,7 @@ import ( // ReconcileApiServerLB tries to move the Load Balancer to the desired OCICluster Spec func (s *ClusterScope) ReconcileApiServerLB(ctx context.Context) error { - desiredApiServerLb := s.LBSpec() + desiredApiServerLb := s.DesiredAPIServerLoadBalancer() lb, err := s.GetLoadBalancers(ctx) if err != nil { @@ -100,14 +100,12 @@ func (s *ClusterScope) DeleteApiServerLB(ctx context.Context) error { return nil } -// LBSpec builds the LoadBalancer from the ClusterScope and returns it. -// Note: `NLBSpec` is the shared API server LB configuration for both OCI NLB and LBaaS paths. -func (s *ClusterScope) LBSpec() infrastructurev1beta2.LoadBalancer { - lbSpec := infrastructurev1beta2.LoadBalancer{ - Name: s.GetControlPlaneLoadBalancerName(), - NLBSpec: s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec, - } - return lbSpec +// DesiredAPIServerLoadBalancer builds the desired LBaaS load balancer from the ClusterScope. +func (s *ClusterScope) DesiredAPIServerLoadBalancer() infrastructurev1beta2.LoadBalancer { + return infrastructurev1beta2.NewAPIServerLoadBalancer( + s.GetControlPlaneLoadBalancerName(), + *s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.APIServerLoadBalancerSpec(), + ) } // GetControlPlaneLoadBalancerName returns the user defined APIServerLB name from the spec or diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index be732d468..b16b97937 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -854,7 +854,7 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { } func (m *MachineScope) desiredAPIServerBackendSetNames() []string { - backendSets := m.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec.CanonicalBackendSets() + backendSets := m.OCIClusterAccessor.GetNetworkSpec().APIServerLB.CanonicalAPIServerBackendSets() names := make([]string, 0, len(backendSets)) for _, backendSet := range backendSets { names = append(names, backendSet.Name) diff --git a/cloud/scope/named_resource_reconcile_helpers.go b/cloud/scope/named_resource_reconcile_helpers.go index dbd5a3d14..d0210ce4a 100644 --- a/cloud/scope/named_resource_reconcile_helpers.go +++ b/cloud/scope/named_resource_reconcile_helpers.go @@ -3,7 +3,7 @@ package scope import infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" func desiredAPIServerBackendSets(lb infrastructurev1beta2.LoadBalancer) []infrastructurev1beta2.NLBBackendSet { - canonicalBackendSets := lb.NLBSpec.CanonicalBackendSets() + canonicalBackendSets := lb.CanonicalAPIServerBackendSets() if len(canonicalBackendSets) > 0 { return canonicalBackendSets } diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index d53a56775..ca0456e08 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -31,7 +31,7 @@ import ( // ReconcileApiServerNLB tries to move the Network Load Balancer to the desired OCICluster Spec func (s *ClusterScope) ReconcileApiServerNLB(ctx context.Context) error { - desiredApiServerNLB := s.NLBSpec() + desiredApiServerNLB := s.DesiredAPIServerNetworkLoadBalancer() nlb, err := s.GetNetworkLoadBalancers(ctx) if err != nil { @@ -97,13 +97,12 @@ func (s *ClusterScope) DeleteApiServerNLB(ctx context.Context) error { return nil } -// NLBSpec builds the Network LoadBalancer from the ClusterScope and returns it -func (s *ClusterScope) NLBSpec() infrastructurev1beta2.LoadBalancer { - nlbSpec := infrastructurev1beta2.LoadBalancer{ - Name: s.GetControlPlaneLoadBalancerName(), - NLBSpec: s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec, - } - return nlbSpec +// DesiredAPIServerNetworkLoadBalancer builds the desired network load balancer from the ClusterScope. +func (s *ClusterScope) DesiredAPIServerNetworkLoadBalancer() infrastructurev1beta2.LoadBalancer { + return infrastructurev1beta2.NewAPIServerLoadBalancer( + s.GetControlPlaneLoadBalancerName(), + *s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.APIServerLoadBalancerSpec(), + ) } // GetControlPlaneLoadBalancerName returns the user defined APIServerLB name from the spec or @@ -319,9 +318,9 @@ func (s *ClusterScope) CreateNLB(ctx context.Context, lb infrastructurev1beta2.L } } var reservedIps []networkloadbalancer.ReservedIp - if len(lb.NLBSpec.ReservedIpIds) > 0 { + if len(lb.APIServerLoadBalancerSpec().ReservedIpIds) > 0 { // since max is one we only take the first ip id supplied - reservedIps = append(reservedIps, networkloadbalancer.ReservedIp{Id: common.String(lb.NLBSpec.ReservedIpIds[0])}) + reservedIps = append(reservedIps, networkloadbalancer.ReservedIp{Id: common.String(lb.APIServerLoadBalancerSpec().ReservedIpIds[0])}) } if len(controlPlaneEndpointSubnets) < 1 { diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index f3a4ea9af..fc5fc1b8f 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -938,7 +938,7 @@ func verifyAdditionalAPIServerBackendSetResources(ctx context.Context, specName, return fmt.Errorf("api server load balancer ID not yet assigned") } - expectedBackendSets := ociCluster.Spec.NetworkSpec.APIServerLB.NLBSpec.CanonicalBackendSets() + expectedBackendSets := ociCluster.Spec.NetworkSpec.APIServerLB.CanonicalAPIServerBackendSets() if len(expectedBackendSets) < 2 { return fmt.Errorf("expected multiple API server backend sets, got %d", len(expectedBackendSets)) } From cabce51037212328518301e15cfde37c98931216 Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 20:36:47 -0400 Subject: [PATCH 28/31] Revert "Align E2E auth with AUTH_CONFIG_DIR" This reverts commit 7016234b2a67b372581026582eda775af9360cb3. --- argo/capoci-e2e-workflow.yaml | 3 - argo/capoci-e2e-workflowtemplate.yaml | 3 - scripts/ci-e2e-preflight.sh | 56 +-------- scripts/ci-e2e.sh | 2 +- scripts/run-e2e.sh | 91 -------------- test/README.md | 53 ++++---- test/e2e/e2e_suite_test.go | 167 +++++++------------------- 7 files changed, 69 insertions(+), 306 deletions(-) delete mode 100755 scripts/run-e2e.sh diff --git a/argo/capoci-e2e-workflow.yaml b/argo/capoci-e2e-workflow.yaml index 419ebeeea..50c22b2a0 100644 --- a/argo/capoci-e2e-workflow.yaml +++ b/argo/capoci-e2e-workflow.yaml @@ -288,9 +288,6 @@ spec: echo "Argo CAPOCI E2E only supports instance principal auth for OCIR login and suite execution." >&2 exit 1 fi - export AUTH_CONFIG_DIR=/workspace/auth-config - mkdir -p "${AUTH_CONFIG_DIR}" - printf '%s' true > "${AUTH_CONFIG_DIR}/useInstancePrincipal" export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "${USE_INSTANCE_PRINCIPAL}" | base64 | tr -d '\n')" # Setup OCIR token and login diff --git a/argo/capoci-e2e-workflowtemplate.yaml b/argo/capoci-e2e-workflowtemplate.yaml index 358afb454..ac949673c 100644 --- a/argo/capoci-e2e-workflowtemplate.yaml +++ b/argo/capoci-e2e-workflowtemplate.yaml @@ -329,9 +329,6 @@ spec: echo "Argo CAPOCI E2E only supports instance principal auth for OCIR login and suite execution." >&2 exit 1 fi - export AUTH_CONFIG_DIR=/workspace/auth-config - mkdir -p "${AUTH_CONFIG_DIR}" - printf '%s' true > "${AUTH_CONFIG_DIR}/useInstancePrincipal" export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "${USE_INSTANCE_PRINCIPAL}" | base64 | tr -d '\n')" # Setup OCIR token and login using instance principal diff --git a/scripts/ci-e2e-preflight.sh b/scripts/ci-e2e-preflight.sh index 82ce4440f..3b99e710d 100755 --- a/scripts/ci-e2e-preflight.sh +++ b/scripts/ci-e2e-preflight.sh @@ -120,58 +120,6 @@ check_base64_env() { fi } -check_dir_file() { - local dir="$1" - local name="$2" - if [ -f "${dir}/${name}" ]; then - note_ok "auth config file ${dir}/${name} is present" - else - note_fail "auth config file ${dir}/${name} is missing" - fi -} - -check_auth_config_dir() { - local dir="$1" - local use_instance_principal="" - local use_session_token="" - - if [ ! -d "${dir}" ]; then - note_fail "AUTH_CONFIG_DIR ${dir} does not exist" - return - fi - - check_dir_file "${dir}" "useInstancePrincipal" - if [ ! -f "${dir}/useInstancePrincipal" ]; then - return - fi - - use_instance_principal="$(tr -d '[:space:]' < "${dir}/useInstancePrincipal")" - if [ "${use_instance_principal}" = "true" ]; then - note_ok "AUTH_CONFIG_DIR selects instance principal" - return - fi - - check_dir_file "${dir}" "useSessionToken" - if [ -f "${dir}/useSessionToken" ]; then - use_session_token="$(tr -d '[:space:]' < "${dir}/useSessionToken")" - fi - - check_dir_file "${dir}" "region" - check_dir_file "${dir}" "tenancy" - check_dir_file "${dir}" "fingerprint" - if [ "${use_session_token}" = "true" ]; then - note_ok "AUTH_CONFIG_DIR selects session token auth" - check_dir_file "${dir}" "sessionToken" - check_dir_file "${dir}" "sessionPrivateKey" - return - fi - - note_ok "AUTH_CONFIG_DIR selects user principal auth" - check_dir_file "${dir}" "user" - check_dir_file "${dir}" "key" - check_dir_file "${dir}" "passphrase" -} - uses_instance_principal_local() { local decoded="" if [ -n "${USE_INSTANCE_PRINCIPAL_B64:-}" ] && decoded=$(decode_base64 "${USE_INSTANCE_PRINCIPAL_B64}"); then @@ -212,9 +160,7 @@ check_focus_specific_envs() { check_local_auth() { if [ -n "${AUTH_CONFIG_DIR:-}" ]; then - note_ok "AUTH_CONFIG_DIR is set" - check_auth_config_dir "${AUTH_CONFIG_DIR}" - return + echo "WARN: AUTH_CONFIG_DIR is set, but the CAPOCI E2E suite does not read AUTH_CONFIG_DIR today." >&2 fi if uses_instance_principal_local; then diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index d2b419b61..1503b1d00 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -62,4 +62,4 @@ if [ -z "${OCI_SSH_KEY}" ]; then export OCI_SSH_KEY fi -scripts/run-e2e.sh --build-images +make test-e2e diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh deleted file mode 100755 index ed6b31de3..000000000 --- a/scripts/run-e2e.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash -# Copyright (c) 2021, 2022 Oracle and/or its affiliates. - -set -o errexit -set -o nounset -set -o pipefail - -RUN_TARGET="test-e2e-run" -PREFLIGHT_ONLY="false" - -usage() { - cat <<'EOF' -Usage: scripts/run-e2e.sh [options] - -Supported local runner for CAPOCI E2E execution. - -Options: - --auth-config-dir AUTH_CONFIG_DIR to use for OCI auth. - --focus Override Ginkgo focus. - --skip Override Ginkgo skip. - --artifacts-dir Override artifact output directory. - --ginkgo-nodes Override Ginkgo parallelism. - --existing-cluster Reuse the current management cluster. - --skip-cleanup Preserve resources after the run. - --build-images Use make test-e2e instead of make test-e2e-run. - --preflight-only Run preflight and stop. - -h, --help Show this help message. -EOF -} - -while [ $# -gt 0 ]; do - case "$1" in - --auth-config-dir) - if [ $# -lt 2 ]; then - echo "missing value for --auth-config-dir" >&2 - exit 1 - fi - export AUTH_CONFIG_DIR="$2" - shift 2 - ;; - --focus) - export GINKGO_FOCUS="$2" - shift 2 - ;; - --skip) - export GINKGO_SKIP="$2" - shift 2 - ;; - --artifacts-dir) - export ARTIFACTS="$2" - shift 2 - ;; - --ginkgo-nodes) - export GINKGO_NODES="$2" - shift 2 - ;; - --existing-cluster) - export SKIP_CREATE_MGMT_CLUSTER=true - shift - ;; - --skip-cleanup) - export SKIP_CLEANUP=true - shift - ;; - --build-images) - RUN_TARGET="test-e2e" - shift - ;; - --preflight-only) - PREFLIGHT_ONLY="true" - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "unknown argument: $1" >&2 - usage >&2 - exit 1 - ;; - esac -done - -make test-e2e-preflight - -if [ "${PREFLIGHT_ONLY}" = "true" ]; then - exit 0 -fi - -make "${RUN_TARGET}" diff --git a/test/README.md b/test/README.md index 836e533be..5cb2b3881 100644 --- a/test/README.md +++ b/test/README.md @@ -6,9 +6,10 @@ This is the source-of-truth runbook for CAPOCI E2E execution in this repo. The E2E suite currently reads auth from `test/e2e/e2e_suite_test.go`. -- Preferred local and Argo auth uses `AUTH_CONFIG_DIR` and `cloud/config.FromDir`. -- Legacy local base64 auth envs still work as a compatibility fallback when `AUTH_CONFIG_DIR` is unset. +- Local instance principal uses `USE_INSTANCE_PRINCIPAL_B64`. +- Local user principal uses base64-encoded env vars such as `OCI_TENANCY_ID_B64`. - Argo currently supports only instance principal for both suite execution and OCIR login. +- `AUTH_CONFIG_DIR` is not read by the E2E suite today. ## Default suite behavior @@ -82,62 +83,60 @@ The local preflight checks: - required tools: `go`, `docker`, `kind`, `kubectl`, `kustomize`, `envsubst`, `jq` - required OCI image inputs - scenario-specific env vars inferred from `GINKGO_FOCUS` -- whether `AUTH_CONFIG_DIR` or the legacy fallback envs are configured for the current suite +- whether auth is configured for the current suite ### 2. Configure auth -Preferred local auth is `AUTH_CONFIG_DIR`. - For local instance principal: ```bash -mkdir -p /tmp/capoci-auth -printf '%s' true > /tmp/capoci-auth/useInstancePrincipal -export AUTH_CONFIG_DIR=/tmp/capoci-auth +export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "true" | base64 | tr -d '\n')" ``` For local user principal: ```bash -mkdir -p /tmp/capoci-auth -printf '%s' false > /tmp/capoci-auth/useInstancePrincipal -printf '%s' false > /tmp/capoci-auth/useSessionToken -printf '%s' > /tmp/capoci-auth/region -printf '%s' > /tmp/capoci-auth/tenancy -printf '%s' > /tmp/capoci-auth/user -printf '%s' > /tmp/capoci-auth/fingerprint -cp /path/to/api-private-key.pem /tmp/capoci-auth/key -printf '%s' > /tmp/capoci-auth/passphrase -export AUTH_CONFIG_DIR=/tmp/capoci-auth +export OCI_TENANCY_ID= +export OCI_USER_ID= +export OCI_CREDENTIALS_FINGERPRINT= +export OCI_REGION= +export OCI_TENANCY_ID_B64="$(printf '%s' "$OCI_TENANCY_ID" | base64 | tr -d '\n')" +export OCI_USER_ID_B64="$(printf '%s' "$OCI_USER_ID" | base64 | tr -d '\n')" +export OCI_CREDENTIALS_FINGERPRINT_B64="$(printf '%s' "$OCI_CREDENTIALS_FINGERPRINT" | base64 | tr -d '\n')" +export OCI_REGION_B64="$(printf '%s' "$OCI_REGION" | base64 | tr -d '\n')" +export OCI_CREDENTIALS_KEY_B64="$(base64 < /path/to/api-private-key.pem | tr -d '\n')" +export OCI_CREDENTIALS_PASSPHRASE_B64="$(printf '%s' "$OCI_CREDENTIALS_PASSPHRASE" | base64 | tr -d '\n')" ``` Notes: -- Legacy base64 envs still work if `AUTH_CONFIG_DIR` is unset. -- Session-token auth also works through `AUTH_CONFIG_DIR` if the directory contains `useSessionToken`, `sessionToken`, and `sessionPrivateKey`. +- Session-token envs are not consumed by `test/e2e/e2e_suite_test.go`. +- `AUTH_CONFIG_DIR` does not configure the E2E suite today. ### 3. Run the suite -Preferred local runner: +Full local flow, including image build and push: ```bash -scripts/run-e2e.sh --auth-config-dir "${AUTH_CONFIG_DIR}" --focus "PRBlocking" +make test-e2e-preflight +REGISTRY= scripts/ci-e2e.sh ``` -Full local flow, including image build and push: +Focused run that reuses the already-built manager image flow: ```bash -REGISTRY= scripts/ci-e2e.sh +make test-e2e-preflight +make test-e2e-run GINKGO_FOCUS="PRBlocking" GINKGO_SKIP="Bare Metal|Multi-Region|VCNPeering" ``` Examples: ```bash -scripts/run-e2e.sh --auth-config-dir "${AUTH_CONFIG_DIR}" --focus "VCNPeering" --skip "" +make test-e2e-run GINKGO_SKIP="" GINKGO_FOCUS="VCNPeering" ``` ```bash -scripts/run-e2e.sh --auth-config-dir "${AUTH_CONFIG_DIR}" --existing-cluster --artifacts-dir "$(pwd)/_artifacts-existing" +make test-e2e-run GINKGO_FOCUS="Conformance" E2E_ARGS='-kubetest.config-file=$(pwd)/test/e2e/data/kubetest/conformance.yaml' ``` Artifacts: @@ -193,8 +192,6 @@ argo submit argo/capoci-e2e-workflow.yaml \ -p use_instance_principal=true ``` -The workflow writes a temporary `AUTH_CONFIG_DIR` inside the job before running `scripts/ci-e2e.sh`. - You can also install and submit from the workflow template: ```bash diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a60552112..1bf4f990c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -193,8 +193,48 @@ var _ = SynchronizedBeforeSuite(func() []byte { e2eConfig = loadE2EConfig(configPath) bootstrapClusterProxy = NewOCIClusterProxy("bootstrap", kubeconfigPath, initScheme()) - ociAuthConfigProvider, err := newE2EAuthConfigProvider() - Expect(err).NotTo(HaveOccurred()) + s, useInstancePrincipalFlagSet := os.LookupEnv("USE_INSTANCE_PRINCIPAL_B64") + useInstanePrincipal := false + if useInstancePrincipalFlagSet { + useInstanePrincipalStr, err := base64.StdEncoding.DecodeString(s) + Expect(err).NotTo(HaveOccurred()) + useInstanePrincipal, err = strconv.ParseBool(string(useInstanePrincipalStr)) + Expect(err).NotTo(HaveOccurred()) + } + var ociAuthConfigProvider common.ConfigurationProvider + var err error + if useInstanePrincipal { + ociAuthConfigProvider, err = oci_config.NewConfigurationProvider(&oci_config.AuthConfig{ + UseInstancePrincipals: true, + }) + Expect(err).NotTo(HaveOccurred()) + By("Using instance principal as auth provider") + } else { + tenancyId, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_TENANCY_ID_B64")) + Expect(err).NotTo(HaveOccurred()) + userId, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_USER_ID_B64")) + Expect(err).NotTo(HaveOccurred()) + passphrase, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_CREDENTIALS_PASSPHRASE_B64")) + Expect(err).NotTo(HaveOccurred()) + key, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_CREDENTIALS_KEY_B64")) + Expect(err).NotTo(HaveOccurred()) + fingerprint, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_CREDENTIALS_FINGERPRINT_B64")) + Expect(err).NotTo(HaveOccurred()) + region, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_REGION_B64")) + Expect(err).NotTo(HaveOccurred()) + + ociAuthConfigProvider, err = oci_config.NewConfigurationProvider(&oci_config.AuthConfig{ + Region: string(region), + TenancyID: string(tenancyId), + UserID: string(userId), + PrivateKey: string(key), + Fingerprint: string(fingerprint), + Passphrase: string(passphrase), + UseInstancePrincipals: false, + }) + Expect(err).NotTo(HaveOccurred()) + By("Using user principal as auth provider") + } clientProvider, err := scope.NewClientProvider(scope.ClientProviderParams{OciAuthConfigProvider: ociAuthConfigProvider}) Expect(err).NotTo(HaveOccurred()) @@ -219,129 +259,6 @@ var _ = SynchronizedBeforeSuite(func() []byte { adCount = len(resp.Items) }) -func newE2EAuthConfigProvider() (common.ConfigurationProvider, error) { - if authConfigDir := os.Getenv("AUTH_CONFIG_DIR"); authConfigDir != "" { - authConfig, err := oci_config.FromDir(authConfigDir) - if err != nil { - return nil, err - } - provider, err := oci_config.NewConfigurationProvider(authConfig) - if err != nil { - return nil, err - } - switch { - case authConfig.UseInstancePrincipals: - By(fmt.Sprintf("Using auth provider from AUTH_CONFIG_DIR %q with instance principal", authConfigDir)) - case authConfig.UseSessionToken: - By(fmt.Sprintf("Using auth provider from AUTH_CONFIG_DIR %q with session token", authConfigDir)) - default: - By(fmt.Sprintf("Using auth provider from AUTH_CONFIG_DIR %q with user principal", authConfigDir)) - } - return provider, nil - } - - useInstancePrincipal, err := useInstancePrincipalFromEnv() - if err != nil { - return nil, err - } - if useInstancePrincipal { - provider, err := oci_config.NewConfigurationProvider(&oci_config.AuthConfig{ - UseInstancePrincipals: true, - }) - if err != nil { - return nil, err - } - By("Using instance principal as auth provider from legacy env vars") - return provider, nil - } - - authConfig, err := legacyBase64AuthConfigFromEnv() - if err != nil { - return nil, err - } - provider, err := oci_config.NewConfigurationProvider(authConfig) - if err != nil { - return nil, err - } - By("Using user principal as auth provider from legacy env vars") - return provider, nil -} - -func useInstancePrincipalFromEnv() (bool, error) { - if rawValue, ok := os.LookupEnv("USE_INSTANCE_PRINCIPAL"); ok && rawValue != "" { - return strconv.ParseBool(rawValue) - } - if encodedValue, ok := os.LookupEnv("USE_INSTANCE_PRINCIPAL_B64"); ok && encodedValue != "" { - decodedValue, err := base64.StdEncoding.DecodeString(encodedValue) - if err != nil { - return false, err - } - return strconv.ParseBool(string(decodedValue)) - } - return false, nil -} - -func legacyBase64AuthConfigFromEnv() (*oci_config.AuthConfig, error) { - tenancyID, err := decodeRequiredBase64Env("OCI_TENANCY_ID_B64") - if err != nil { - return nil, err - } - userID, err := decodeRequiredBase64Env("OCI_USER_ID_B64") - if err != nil { - return nil, err - } - privateKey, err := decodeRequiredBase64Env("OCI_CREDENTIALS_KEY_B64") - if err != nil { - return nil, err - } - fingerprint, err := decodeRequiredBase64Env("OCI_CREDENTIALS_FINGERPRINT_B64") - if err != nil { - return nil, err - } - region, err := decodeRequiredBase64Env("OCI_REGION_B64") - if err != nil { - return nil, err - } - passphrase, err := decodeOptionalBase64Env("OCI_CREDENTIALS_PASSPHRASE_B64") - if err != nil { - return nil, err - } - - return &oci_config.AuthConfig{ - Region: region, - TenancyID: tenancyID, - UserID: userID, - PrivateKey: privateKey, - Fingerprint: fingerprint, - Passphrase: passphrase, - UseInstancePrincipals: false, - }, nil -} - -func decodeRequiredBase64Env(name string) (string, error) { - value := os.Getenv(name) - if value == "" { - return "", fmt.Errorf("%s is not set", name) - } - decoded, err := base64.StdEncoding.DecodeString(value) - if err != nil { - return "", err - } - return string(decoded), nil -} - -func decodeOptionalBase64Env(name string) (string, error) { - value := os.Getenv(name) - if value == "" { - return "", nil - } - decoded, err := base64.StdEncoding.DecodeString(value) - if err != nil { - return "", err - } - return string(decoded), nil -} - // Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads). // The bootstrap cluster is shared across all the tests, so it should be deleted only after all ParallelNodes completes. // The local clusterctl repository is preserved like everything else created into the artifact folder. From a238ca394e1bad6f6af8e1992c814cd70ccf7f7f Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 20:36:57 -0400 Subject: [PATCH 29/31] Revert "Add CAPOCI E2E runbook and preflight" This reverts commit 507d2690a5a04dd5bb08a8590743d098ba2753b8. --- Makefile | 4 - scripts/ci-e2e-preflight.sh | 227 ------------------------------ test/README.md | 266 +++++++++--------------------------- 3 files changed, 63 insertions(+), 434 deletions(-) delete mode 100755 scripts/ci-e2e-preflight.sh diff --git a/Makefile b/Makefile index d01ffe212..45c8d56e7 100644 --- a/Makefile +++ b/Makefile @@ -313,10 +313,6 @@ test-e2e-run: generate-e2e-templates $(GINKGO) $(ENVSUBST) ## Run e2e tests -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) -.PHONY: test-e2e-preflight -test-e2e-preflight: ## Validate CAPOCI e2e prerequisites before a full run - ./scripts/ci-e2e-preflight.sh $(E2E_PREFLIGHT_ARGS) - .PHONY: test-e2e test-e2e: ## Run e2e tests PULL_POLICY=IfNotPresent MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \ diff --git a/scripts/ci-e2e-preflight.sh b/scripts/ci-e2e-preflight.sh deleted file mode 100755 index 3b99e710d..000000000 --- a/scripts/ci-e2e-preflight.sh +++ /dev/null @@ -1,227 +0,0 @@ -#!/bin/bash -# Copyright (c) 2021, 2022 Oracle and/or its affiliates. - -set -o errexit -set -o nounset -set -o pipefail - -MODE="local" - -usage() { - cat <<'EOF' -Usage: scripts/ci-e2e-preflight.sh [--mode local|argo] - -Validates CAPOCI E2E prerequisites before running a full suite. - -Modes: - local Validate local scripts/ci-e2e.sh execution inputs. - argo Validate Argo submission inputs and instance-principal-only auth. -EOF -} - -while [ $# -gt 0 ]; do - case "$1" in - --mode) - if [ $# -lt 2 ]; then - echo "missing value for --mode" >&2 - exit 1 - fi - MODE="$2" - shift 2 - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "unknown argument: $1" >&2 - usage >&2 - exit 1 - ;; - esac -done - -case "${MODE}" in - local|argo) - ;; - *) - echo "unsupported mode: ${MODE}" >&2 - usage >&2 - exit 1 - ;; -esac - -failures=0 - -note_ok() { - echo "OK: $1" -} - -note_fail() { - echo "FAIL: $1" >&2 - failures=$((failures + 1)) -} - -check_command() { - local name="$1" - if command -v "${name}" >/dev/null 2>&1; then - note_ok "found command ${name}" - else - note_fail "missing command ${name}" - fi -} - -check_file() { - local path="$1" - if [ -f "${path}" ]; then - note_ok "found file ${path}" - else - note_fail "missing file ${path}" - fi -} - -check_env() { - local name="$1" - if [ -n "${!name:-}" ]; then - note_ok "env ${name} is set" - else - note_fail "env ${name} is not set" - fi -} - -decode_base64() { - local value="$1" - if decoded=$(printf '%s' "${value}" | base64 --decode 2>/dev/null); then - printf '%s' "${decoded}" - return 0 - fi - if decoded=$(printf '%s' "${value}" | base64 -D 2>/dev/null); then - printf '%s' "${decoded}" - return 0 - fi - return 1 -} - -check_base64_env() { - local name="$1" - local decoded="" - if [ -z "${!name:-}" ]; then - note_fail "env ${name} is not set" - return - fi - if decoded=$(decode_base64 "${!name}"); then - if [ -n "${decoded}" ]; then - note_ok "env ${name} is set and base64 decodes" - else - note_fail "env ${name} decodes to an empty value" - fi - else - note_fail "env ${name} is not valid base64" - fi -} - -uses_instance_principal_local() { - local decoded="" - if [ -n "${USE_INSTANCE_PRINCIPAL_B64:-}" ] && decoded=$(decode_base64 "${USE_INSTANCE_PRINCIPAL_B64}"); then - [ "${decoded}" = "true" ] - return - fi - return 1 -} - -check_core_envs() { - check_env OCI_COMPARTMENT_ID - check_env OCI_IMAGE_ID - check_env OCI_ORACLE_LINUX_IMAGE_ID - check_env OCI_UPGRADE_IMAGE_ID - check_env OCI_ALTERNATIVE_REGION_IMAGE_ID - check_env OCI_MANAGED_NODE_IMAGE_ID -} - -check_focus_specific_envs() { - local focus="${GINKGO_FOCUS:-PRBlocking}" - if [[ "${focus}" == *Windows* ]]; then - check_env OCI_WINDOWS_IMAGE_ID - fi - if [[ "${focus}" == *VCNPeering* ]]; then - check_env LOCAL_DRG_ID - check_env PEER_DRG_ID - check_env PEER_REGION_NAME - check_env OCI_ALTERNATIVE_REGION - check_env EXTERNAL_VCN_ID - check_env EXTERNAL_VCN_CPE_NSG - check_env EXTERNAL_VCN_WORKER_NSG - check_env EXTERNAL_VCN_CP_NSG - check_env EXTERNAL_VCN_CPE_SUBNET - check_env EXTERNAL_VCN_WORKER_SUBNET - check_env EXTERNAL_VCN_CP_SUBNET - fi -} - -check_local_auth() { - if [ -n "${AUTH_CONFIG_DIR:-}" ]; then - echo "WARN: AUTH_CONFIG_DIR is set, but the CAPOCI E2E suite does not read AUTH_CONFIG_DIR today." >&2 - fi - - if uses_instance_principal_local; then - note_ok "local auth is configured for instance principal via USE_INSTANCE_PRINCIPAL_B64" - return - fi - - check_base64_env OCI_TENANCY_ID_B64 - check_base64_env OCI_USER_ID_B64 - check_base64_env OCI_CREDENTIALS_KEY_B64 - check_base64_env OCI_CREDENTIALS_FINGERPRINT_B64 - check_base64_env OCI_REGION_B64 - if [ -n "${OCI_CREDENTIALS_PASSPHRASE_B64:-}" ]; then - check_base64_env OCI_CREDENTIALS_PASSPHRASE_B64 - fi - - if [ -n "${OCI_SESSION_TOKEN_B64:-}" ] || [ -n "${OCI_SESSION_PRIVATE_KEY_B64:-}" ]; then - note_fail "session-token auth envs are not consumed by test/e2e/e2e_suite_test.go" - fi -} - -check_argo_auth() { - if [ "${USE_INSTANCE_PRINCIPAL:-}" = "true" ]; then - note_ok "argo auth is configured for instance principal" - else - note_fail "argo workflow currently supports only USE_INSTANCE_PRINCIPAL=true" - fi -} - -echo "Running CAPOCI E2E preflight in ${MODE} mode" - -check_file "scripts/ci-e2e.sh" -check_file "test/e2e/config/e2e_conf.yaml" - -if [ "${MODE}" = "local" ]; then - check_command go - check_command docker - check_command kind - check_command kubectl - check_command kustomize - check_command envsubst - check_command jq - check_core_envs - check_focus_specific_envs - check_local_auth -else - check_command argo - check_command kubectl - check_command jq - check_file "argo/capoci-e2e-workflow.yaml" - check_file "argo/capoci-e2e-workflowtemplate.yaml" - check_file "argo/capoci-e2e-configmap.yaml" - check_core_envs - check_focus_specific_envs - check_env REGISTRY - check_argo_auth -fi - -if [ "${failures}" -ne 0 ]; then - echo "CAPOCI E2E preflight failed with ${failures} issue(s)." >&2 - exit 1 -fi - -echo "CAPOCI E2E preflight passed." diff --git a/test/README.md b/test/README.md index 5cb2b3881..4b75b2443 100644 --- a/test/README.md +++ b/test/README.md @@ -1,206 +1,66 @@ -# CAPOCI E2E Runbook - -This is the source-of-truth runbook for CAPOCI E2E execution in this repo. - -## Current auth model - -The E2E suite currently reads auth from `test/e2e/e2e_suite_test.go`. - -- Local instance principal uses `USE_INSTANCE_PRINCIPAL_B64`. -- Local user principal uses base64-encoded env vars such as `OCI_TENANCY_ID_B64`. -- Argo currently supports only instance principal for both suite execution and OCIR login. -- `AUTH_CONFIG_DIR` is not read by the E2E suite today. - -## Default suite behavior - -- `make test-e2e` builds and pushes the manager image, then runs the E2E suite. -- `make test-e2e-run` skips the image build and push step. -- Default focus is `PRBlocking`. -- Default skip is `Bare Metal|Multi-Region|VCNPeering`. -- Default Ginkgo parallelism is `GINKGO_NODES=3`. - -## Required env vars for standard runs - -These are required for both local and Argo execution: - -```bash -export OCI_COMPARTMENT_ID= -export OCI_IMAGE_ID= -export OCI_ORACLE_LINUX_IMAGE_ID= -export OCI_UPGRADE_IMAGE_ID= -export OCI_ALTERNATIVE_REGION_IMAGE_ID= -export OCI_MANAGED_NODE_IMAGE_ID= -``` - -Optional standard inputs: - -```bash -export OCI_WINDOWS_IMAGE_ID= -export OCI_SSH_KEY="ssh-rsa AAAA..." -export TAG= -export GINKGO_NODES=3 -export GINKGO_FOCUS="PRBlocking" -export GINKGO_SKIP="Bare Metal|Multi-Region|VCNPeering" -export REGISTRY= -``` - -## Scenario-specific env vars - -Windows-focused runs also require: - -```bash -export OCI_WINDOWS_IMAGE_ID= -``` - -VCN peering runs also require: - +#Prerequisite +- Install `envsubst` +- Install `kustomize` +- If you have a running `kind` cluster, please delete the `kind` cluster + +# Export the following auth settings if user principal is used as credential + ```bash + export OCI_TENANCY_ID= + export OCI_USER_ID= + export OCI_CREDENTIALS_FINGERPRINT= + export OCI_REGION= + # if Passphrase is present + export OCI_CREDENTIALS_PASSPHRASE= + export OCI_TENANCY_ID_B64="$(echo -n "$OCI_TENANCY_ID" | base64 | tr -d '\n')" + export OCI_CREDENTIALS_FINGERPRINT_B64="$(echo -n "$OCI_CREDENTIALS_FINGERPRINT" | base64 | tr -d '\n')" + export OCI_USER_ID_B64="$(echo -n "$OCI_USER_ID" | base64 | tr -d '\n')" + export OCI_REGION_B64="$(echo -n "$OCI_REGION" | base64 | tr -d '\n')" + export OCI_CREDENTIALS_KEY_B64=$( cat | base64 | tr -d '\n' ) + # if Passphrase is present + export OCI_CREDENTIALS_PASSPHRASE_B64="$(echo -n "$OCI_CREDENTIALS_PASSPHRASE" | base64 | tr -d '\n')" + ``` + +# Export the following auth settings if instance principal has to be used + ```bash + export USE_INSTANCE_PRINCIPAL_B64="$(echo -n "true" | base64 | tr -d '\n')" + ``` + + + +# Export the following test settings + ```bash + export OCI_COMPARTMENT_ID=<> + export OCI_IMAGE_ID=<> + export OCI_UPGRADE_IMAGE_ID= + export OCI_ORACLE_LINUX_IMAGE_ID= + export OCI_MANAGED_NODE_IMAGE_ID= + ``` +# Execute the test script + ```bash + export REGISTRY="iad.ocir.io/" + export LOCAL_ONLY=false + scripts/ci-e2e.sh + ``` + +# For the VCN Peering related tests, the following parameters needs to be set + ```bash + export LOCAL_DRG_ID= + export OCI_ALTERNATIVE_REGION_IMAGE_ID= + export OCI_ALTERNATIVE_REGION= + export PEER_REGION_NAME= + export PEER_DRG_ID= + ``` + +Then runt he test script by setting the correct focus parameter like below ```bash -export LOCAL_DRG_ID= -export PEER_DRG_ID= -export PEER_REGION_NAME= -export OCI_ALTERNATIVE_REGION= -export EXTERNAL_VCN_ID= -export EXTERNAL_VCN_CPE_NSG= -export EXTERNAL_VCN_WORKER_NSG= -export EXTERNAL_VCN_CP_NSG= -export EXTERNAL_VCN_CPE_SUBNET= -export EXTERNAL_VCN_WORKER_SUBNET= -export EXTERNAL_VCN_CP_SUBNET= +GINKGO_SKIP="" GINKGO_FOCUS="VCNPeering" scripts/ci-e2e.sh ``` -## Local execution - -### 1. Validate prerequisites - -Run the preflight before a full suite: - -```bash -make test-e2e-preflight -``` - -The local preflight checks: - -- required tools: `go`, `docker`, `kind`, `kubectl`, `kustomize`, `envsubst`, `jq` -- required OCI image inputs -- scenario-specific env vars inferred from `GINKGO_FOCUS` -- whether auth is configured for the current suite - -### 2. Configure auth - -For local instance principal: - -```bash -export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "true" | base64 | tr -d '\n')" -``` - -For local user principal: - -```bash -export OCI_TENANCY_ID= -export OCI_USER_ID= -export OCI_CREDENTIALS_FINGERPRINT= -export OCI_REGION= -export OCI_TENANCY_ID_B64="$(printf '%s' "$OCI_TENANCY_ID" | base64 | tr -d '\n')" -export OCI_USER_ID_B64="$(printf '%s' "$OCI_USER_ID" | base64 | tr -d '\n')" -export OCI_CREDENTIALS_FINGERPRINT_B64="$(printf '%s' "$OCI_CREDENTIALS_FINGERPRINT" | base64 | tr -d '\n')" -export OCI_REGION_B64="$(printf '%s' "$OCI_REGION" | base64 | tr -d '\n')" -export OCI_CREDENTIALS_KEY_B64="$(base64 < /path/to/api-private-key.pem | tr -d '\n')" -export OCI_CREDENTIALS_PASSPHRASE_B64="$(printf '%s' "$OCI_CREDENTIALS_PASSPHRASE" | base64 | tr -d '\n')" -``` - -Notes: - -- Session-token envs are not consumed by `test/e2e/e2e_suite_test.go`. -- `AUTH_CONFIG_DIR` does not configure the E2E suite today. - -### 3. Run the suite - -Full local flow, including image build and push: - -```bash -make test-e2e-preflight -REGISTRY= scripts/ci-e2e.sh -``` - -Focused run that reuses the already-built manager image flow: - -```bash -make test-e2e-preflight -make test-e2e-run GINKGO_FOCUS="PRBlocking" GINKGO_SKIP="Bare Metal|Multi-Region|VCNPeering" -``` - -Examples: - -```bash -make test-e2e-run GINKGO_SKIP="" GINKGO_FOCUS="VCNPeering" -``` - -```bash -make test-e2e-run GINKGO_FOCUS="Conformance" E2E_ARGS='-kubetest.config-file=$(pwd)/test/e2e/data/kubetest/conformance.yaml' -``` - -Artifacts: - -- local logs and manifests are written under `_artifacts/` -- rendered config is written to `test/e2e/config/e2e_conf-envsubst.yaml` - -## Argo execution - -### 1. Validate inputs - -Run the Argo-oriented preflight: - -```bash -make test-e2e-preflight E2E_PREFLIGHT_ARGS="--mode argo" -``` - -The Argo preflight checks: - -- required tools: `argo`, `kubectl`, `jq` -- workflow/config manifests exist in the repo -- required OCI image inputs are exported -- `REGISTRY` is set for submission -- `USE_INSTANCE_PRINCIPAL=true` is selected - -### 2. Prepare the config map - -Update `argo/capoci-e2e-configmap.yaml` with the required OCI image IDs and registry, then apply it: - -```bash -kubectl apply -f argo/capoci-e2e-configmap.yaml -``` - -### 3. Submit the workflow - -Argo currently supports instance principal only: - -```bash -export USE_INSTANCE_PRINCIPAL=true -export REGISTRY= -make test-e2e-preflight E2E_PREFLIGHT_ARGS="--mode argo" - -argo submit argo/capoci-e2e-workflow.yaml \ - -p git_ref= \ - -p git_repo= \ - -p registry="${REGISTRY}" \ - -p oci_compartment_id="${OCI_COMPARTMENT_ID}" \ - -p oci_image_id="${OCI_IMAGE_ID}" \ - -p oci_oracle_linux_image_id="${OCI_ORACLE_LINUX_IMAGE_ID}" \ - -p oci_upgrade_image_id="${OCI_UPGRADE_IMAGE_ID}" \ - -p oci_alternative_region_image_id="${OCI_ALTERNATIVE_REGION_IMAGE_ID}" \ - -p oci_managed_node_image_id="${OCI_MANAGED_NODE_IMAGE_ID}" \ - -p use_instance_principal=true -``` - -You can also install and submit from the workflow template: - -```bash -kubectl apply -f argo/capoci-e2e-workflowtemplate.yaml -argo submit --from workflowtemplate/capoci-e2e -n argo -p use_instance_principal=true -``` - -Argo artifacts: - -- `report.json` -- `_artifacts_logs_only` -- `_artifacts` +For the remote VCN peering test to run, the DRG has to be created manually in management cluster VCN, +and routing rules should have been added in management cluster VCN to route the workload cluster traffic to the DRG. +Following are the steps for the same +1. Create a DRG in the region where test needs to run +2. The assumption is that the VCN on which management cluster runs is already created. Add a VCN attachment in the DRG + as explained in the doc https://docs.oracle.com/en-us/iaas/Content/Network/Tasks/managingDRGs.htm under section "Attaching a VCN to a DRG" +3. Add a routing rule in the VCN Route Rules to forward traffic to workload cluster VCN, in this case, "10.1.0.0/16" + to the DRG. This is explained in the section "To route a subnet's traffic to a DRG" in the above doc. From 231d62047785d53dd492609fe931eb11747a96ff Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 10 Mar 2026 20:37:02 -0400 Subject: [PATCH 30/31] Revert "Fix CAPOCI E2E script and Argo auth drift" This reverts commit 27bf51fe59503d25adb2c725311ee706a88b5146. --- argo/capoci-e2e-workflow.yaml | 18 ++++++++---------- argo/capoci-e2e-workflowtemplate.yaml | 18 ++++++++---------- scripts/ci-e2e.sh | 26 +++++++++----------------- 3 files changed, 25 insertions(+), 37 deletions(-) diff --git a/argo/capoci-e2e-workflow.yaml b/argo/capoci-e2e-workflow.yaml index 50c22b2a0..903f26814 100644 --- a/argo/capoci-e2e-workflow.yaml +++ b/argo/capoci-e2e-workflow.yaml @@ -14,10 +14,11 @@ # -p oci_alternative_region_image_id=ocid1.image.oc1..example \ # -p registry=ghcr.io/your-org \ # -p use_instance_principal=true \ +# -p use_instance_principal_b64=dHJ1ZQ== \ # -p oci_ssh_key="ssh-rsa AAAA... your-key" # # Notes: -# - This workflow supports instance principal for OCIR login and the E2E run. +# - USE_INSTANCE_PRINCIPAL_B64 commonly needs "dHJ1ZQ==" for true, "ZmFsc2U=" for false (base64). # - If OCI_SSH_KEY is omitted, the script will generate a temporary key. # - REGISTRY defaults to ghcr.io/oracle; override if pushing to your org's registry. # - git_ref defaults to main @@ -81,7 +82,10 @@ spec: - name: exp_oke value: "false" - name: use_instance_principal - value: "true" + value: "false" + # Base64 of "true" or "false". Common values: true=dHJ1ZQ==, false=ZmFsc2U= + - name: use_instance_principal_b64 + value: "dHJ1ZQ==" - name: oci_alternative_region value: "us-sanjose-1" @@ -207,7 +211,7 @@ spec: - name: USE_INSTANCE_PRINCIPAL value: "{{workflow.parameters.use_instance_principal}}" - name: USE_INSTANCE_PRINCIPAL_B64 - value: "" + value: "{{workflow.parameters.use_instance_principal_b64}}" # Test runner config - name: GINKGO_NODES @@ -233,7 +237,7 @@ spec: - name: OCIR_REGION value: "{{workflow.parameters.ocir_region}}" - name: OCIR_REGION_SHORT_CODE - value: "{{workflow.parameters.ocir_region_short_code}}" + value: "{{workflow.parameters.ocir_region}}" - name: KIND_CGROUP_DRIVER value: "cgroupfs" @@ -284,12 +288,6 @@ spec: oci --version || true - if [ "${USE_INSTANCE_PRINCIPAL}" != "true" ]; then - echo "Argo CAPOCI E2E only supports instance principal auth for OCIR login and suite execution." >&2 - exit 1 - fi - export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "${USE_INSTANCE_PRINCIPAL}" | base64 | tr -d '\n')" - # Setup OCIR token and login echo "Generating the ocir token" echo "https://{{workflow.parameters.ocir_region}}.ocir.io/20180419/docker/token" diff --git a/argo/capoci-e2e-workflowtemplate.yaml b/argo/capoci-e2e-workflowtemplate.yaml index ac949673c..aae7e176a 100644 --- a/argo/capoci-e2e-workflowtemplate.yaml +++ b/argo/capoci-e2e-workflowtemplate.yaml @@ -25,10 +25,11 @@ # # Run a workflow from this template (override any defaults with -p): # argo submit --from workflowtemplate/capoci-e2e -n argo \ -# -p use_instance_principal=true +# -p use_instance_principal=true \ +# -p use_instance_principal_b64=dHJ1ZQ== # # Notes: -# - This workflow template supports instance principal for OCIR login and the E2E run. +# - USE_INSTANCE_PRINCIPAL_B64 commonly needs "dHJ1ZQ==" for true, "ZmFsc2U=" for false (base64). # - If OCI_SSH_KEY is omitted, the script will generate a temporary key. # - REGISTRY defaults to ghcr.io/oracle; override if pushing to your org's registry. # - git_ref defaults to main @@ -111,7 +112,10 @@ spec: - name: exp_oke value: "false" - name: use_instance_principal - value: "true" + value: "false" + # Base64 of "true" or "false". Common values: true=dHJ1ZQ==, false=ZmFsc2U= + - name: use_instance_principal_b64 + value: "dHJ1ZQ==" - name: oci_alternative_region value: "us-sanjose-1" @@ -243,7 +247,7 @@ spec: - name: USE_INSTANCE_PRINCIPAL value: "{{workflow.parameters.use_instance_principal}}" - name: USE_INSTANCE_PRINCIPAL_B64 - value: "" + value: "{{workflow.parameters.use_instance_principal_b64}}" # Test runner config - name: GINKGO_NODES @@ -325,12 +329,6 @@ spec: oci --version || true - if [ "${USE_INSTANCE_PRINCIPAL}" != "true" ]; then - echo "Argo CAPOCI E2E only supports instance principal auth for OCIR login and suite execution." >&2 - exit 1 - fi - export USE_INSTANCE_PRINCIPAL_B64="$(printf '%s' "${USE_INSTANCE_PRINCIPAL}" | base64 | tr -d '\n')" - # Setup OCIR token and login using instance principal echo "Generating the ocir token" echo "https://{{workflow.parameters.ocir_region}}.ocir.io/20180419/docker/token" diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 1503b1d00..571f01b3b 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -20,28 +20,20 @@ source "${REPO_ROOT}/hack/ensure-kubectl.sh" # shellcheck source=hack/ensure-tags.sh source "${REPO_ROOT}/hack/ensure-tags.sh" -require_env() { - local name="$1" - if [ -z "${!name:-}" ]; then - echo "Environment variable ${name} is empty or not defined." >&2 - exit 1 - fi -} - # Verify the required Environment Variables are present. -require_env OCI_COMPARTMENT_ID -require_env OCI_IMAGE_ID -require_env OCI_ORACLE_LINUX_IMAGE_ID -require_env OCI_UPGRADE_IMAGE_ID -require_env OCI_ALTERNATIVE_REGION_IMAGE_ID -require_env OCI_MANAGED_NODE_IMAGE_ID +: "${OCI_COMPARTMENT_ID:?Environment variable empty or not defined.}" +: "${OCI_IMAGE_ID:?Environment variable empty or not defined.}" +: "${OCI_ORACLE_LINUX_IMAGE_ID:?Environment variable empty or not defined.}" +: "${OCI_UPGRADE_IMAGE_ID:?Environment variable empty or not defined.}" +: "${OCI_ALTERNATIVE_REGION_IMAGE_ID:?Environment variable empty or not defined.}" +: "${OCI_MANAGED_NODE_IMAGE_ID:?Environment variable empty or not defined.}" +: OCI_WINDOWS_IMAGE_ID export LOCAL_ONLY=${LOCAL_ONLY:-"true"} -export OCI_WINDOWS_IMAGE_ID="${OCI_WINDOWS_IMAGE_ID:-""}" defaultTag=$(date -u '+%Y%m%d%H%M%S') -export TAG="${TAG:-${defaultTag}}" -export GINKGO_NODES="${GINKGO_NODES:-3}" +export TAG="${defaultTag:-dev}" +export GINKGO_NODES=3 export OCI_SSH_KEY="${OCI_SSH_KEY:-""}" export OCI_CONTROL_PLANE_MACHINE_TYPE="${OCI_CONTROL_PLANE_MACHINE_TYPE:-"VM.Standard.E3.Flex"}" From 3c5973f88cf4aa08a68823904b18340249037c3b Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Wed, 11 Mar 2026 08:55:07 -0400 Subject: [PATCH 31/31] Fix failing e2e --- cloud/scope/machine.go | 9 ++++++++- cloud/scope/machine_backendsets_test.go | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index b16b97937..492880a60 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -863,7 +863,14 @@ func (m *MachineScope) desiredAPIServerBackendSetNames() []string { } func (m *MachineScope) desiredAPIServerBackendPort(backendSetName string) int32 { - return m.OCIClusterAccessor.GetControlPlaneEndpoint().Port + defaultPort := m.OCIClusterAccessor.GetControlPlaneEndpoint().Port + for _, backendSet := range m.OCIClusterAccessor.GetNetworkSpec().APIServerLB.CanonicalAPIServerBackendSets() { + if backendSet.Name != backendSetName { + continue + } + return desiredAPIServerListenerPort(defaultPort, backendSet) + } + return defaultPort } func (m *MachineScope) createBackendRetryToken(backendSetName string, backendPort int32) *string { diff --git a/cloud/scope/machine_backendsets_test.go b/cloud/scope/machine_backendsets_test.go index ac8986d96..34b935c43 100644 --- a/cloud/scope/machine_backendsets_test.go +++ b/cloud/scope/machine_backendsets_test.go @@ -277,10 +277,10 @@ func TestNLBCreateMembership_DefaultAndSecondaryBackendSetNaming(t *testing.T) { BackendSetName: common.String("apiserver-lb-backendset-2"), CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ IpAddress: common.String("1.1.1.1"), - Port: common.Int(6443), + Port: common.Int(9345), Name: common.String(fmt.Sprintf("test-%s", stableShortHash("apiserver-lb-backendset-2"))), }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("apiserver-lb-backendset-2:6443")), + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("apiserver-lb-backendset-2:9345")), })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-secondary")}, nil) nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ WorkRequestId: common.String("wrid-secondary"),