diff --git a/Makefile b/Makefile index f66b1a23f..45c8d56e7 100644 --- a/Makefile +++ b/Makefile @@ -287,6 +287,8 @@ generate-e2e-templates: $(KUSTOMIZE) $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-custom-networking-seclist --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-custom-networking-seclist.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-custom-networking-nsg --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-custom-networking-nsg.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-node-nsg --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-node-nsg.yaml + $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-nlb --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-nlb.yaml + $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-lb --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-lb.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-cluster-class --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-cluster-class.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-local-vcn-peering --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-local-vcn-peering.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-remote-vcn-peering --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-remote-vcn-peering.yaml diff --git a/api/internal/apiserverlb/backendsets.go b/api/internal/apiserverlb/backendsets.go new file mode 100644 index 000000000..42d2b67de --- /dev/null +++ b/api/internal/apiserverlb/backendsets.go @@ -0,0 +1,153 @@ +package apiserverlb + +import ( + "fmt" + "regexp" + "strings" + + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +const BackendSetNameRegex = "^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$" + +type BackendSet struct { + Name string + ListenerPort *int32 +} + +func CanonicalBackendSets[T any](configured []T, synthesize func() T) []T { + if len(configured) > 0 { + return append([]T(nil), configured...) + } + return []T{synthesize()} +} + +func ValidateBackendSets(backendSets []BackendSet, legacyConfigured bool, apiServerPort *int32, supportedSecondaryListenerPort int32, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + if len(backendSets) > 0 && legacyConfigured { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child("backendSetDetails"), + "cannot be set when backendSets is configured; move legacy backendSetDetails into backendSets", + )) + } + + nameRegex := regexp.MustCompile(BackendSetNameRegex) + seen := map[string]int{} + for i, backendSet := range backendSets { + namePath := fldPath.Child("backendSets").Index(i).Child("name") + name := strings.TrimSpace(backendSet.Name) + if name == "" { + allErrs = append(allErrs, field.Required(namePath, "must not be empty")) + continue + } + if !nameRegex.MatchString(name) { + allErrs = append(allErrs, field.Invalid( + namePath, + backendSet.Name, + "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$ (1-32 chars; alphanumeric, '-', '_')", + )) + } + if firstIdx, ok := seen[name]; ok { + allErrs = append(allErrs, field.Invalid( + namePath, + backendSet.Name, + fmt.Sprintf("duplicate backend set name, already used at backendSets[%d].name", firstIdx), + )) + continue + } + seen[name] = i + } + + usedPorts := map[int32]int{} + omittedListenerPortIndex := -1 + for i, backendSet := range backendSets { + portPath := fldPath.Child("backendSets").Index(i).Child("listenerPort") + if backendSet.ListenerPort != nil { + port := *backendSet.ListenerPort + if port < 1 || port > 65535 { + allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535")) + continue + } + if !IsSupportedAPIServerListenerPort(port, apiServerPort, supportedSecondaryListenerPort) { + allErrs = append(allErrs, field.Invalid( + portPath, + port, + fmt.Sprintf("must be one of %s to avoid exposing arbitrary control-plane ports", SupportedAPIServerListenerPortsString(apiServerPort, supportedSecondaryListenerPort)), + )) + continue + } + } + + port, known := EffectiveAPIServerListenerPort(backendSet.ListenerPort, apiServerPort) + if !known { + if omittedListenerPortIndex >= 0 { + allErrs = append(allErrs, field.Invalid( + portPath, + backendSet.ListenerPort, + fmt.Sprintf("multiple backend sets omit listenerPort, already omitted at backendSets[%d].listenerPort", omittedListenerPortIndex), + )) + continue + } + omittedListenerPortIndex = i + continue + } + + if firstIdx, ok := usedPorts[port]; ok { + allErrs = append(allErrs, field.Invalid( + portPath, + backendSet.ListenerPort, + fmt.Sprintf("duplicate effective listenerPort %d, already used at backendSets[%d].listenerPort", port, firstIdx), + )) + continue + } + usedPorts[port] = i + } + + return allErrs +} + +func EffectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { + if listenerPort != nil { + return *listenerPort, true + } + if apiServerPort == nil { + return 0, false + } + return *apiServerPort, true +} + +func IsSupportedAPIServerListenerPort(port int32, apiServerPort *int32, supportedSecondaryListenerPort int32) bool { + for _, allowedPort := range SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort) { + if port == allowedPort { + return true + } + } + return false +} + +func SupportedAPIServerListenerPorts(apiServerPort *int32, supportedSecondaryListenerPort int32) []int32 { + ports := []int32{ociutil.DefaultAPIServerPort} + if apiServerPort != nil && *apiServerPort != ociutil.DefaultAPIServerPort { + ports = append(ports, *apiServerPort) + } + if supportedSecondaryListenerPort != ociutil.DefaultAPIServerPort { + ports = append(ports, supportedSecondaryListenerPort) + } + return ports +} + +func SupportedAPIServerListenerPortsString(apiServerPort *int32, supportedSecondaryListenerPort int32) string { + values := SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort) + parts := make([]string, 0, len(values)) + seen := map[int32]struct{}{} + for _, value := range values { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + parts = append(parts, fmt.Sprintf("%d", value)) + } + return strings.Join(parts, ", ") +} diff --git a/api/v1beta1/backendsets_shared.go b/api/v1beta1/backendsets_shared.go new file mode 100644 index 000000000..1f4366aad --- /dev/null +++ b/api/v1beta1/backendsets_shared.go @@ -0,0 +1,42 @@ +package v1beta1 + +import ( + apiserverlb "github.com/oracle/cluster-api-provider-oci/api/internal/apiserverlb" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func canonicalAPIServerBackendSets(configured []NLBBackendSet, legacy BackendSetDetails) []NLBBackendSet { + return apiserverlb.CanonicalBackendSets(configured, func() NLBBackendSet { + return NLBBackendSet{ + Name: APIServerLBBackendSetName, + BackendSetDetails: legacy, + } + }) +} + +func validateSharedAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { + backendSets := make([]apiserverlb.BackendSet, 0, len(spec.BackendSets)) + for _, backendSet := range spec.BackendSets { + backendSets = append(backendSets, apiserverlb.BackendSet{ + Name: backendSet.Name, + ListenerPort: backendSet.ListenerPort, + }) + } + return apiserverlb.ValidateBackendSets(backendSets, spec.BackendSetDetails != (BackendSetDetails{}), apiServerPort, supportedSecondaryListenerPort, fldPath) +} + +func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { + return apiserverlb.EffectiveAPIServerListenerPort(listenerPort, apiServerPort) +} + +func isSupportedAPIServerListenerPort(port int32, apiServerPort *int32) bool { + return apiserverlb.IsSupportedAPIServerListenerPort(port, apiServerPort, supportedSecondaryListenerPort) +} + +func supportedAPIServerListenerPorts(apiServerPort *int32) []int32 { + return apiserverlb.SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort) +} + +func supportedAPIServerListenerPortsString(apiServerPort *int32) string { + return apiserverlb.SupportedAPIServerListenerPortsString(apiServerPort, supportedSecondaryListenerPort) +} diff --git a/api/v1beta1/ocicluster_conversion_test.go b/api/v1beta1/ocicluster_conversion_test.go index 787ccd8a3..005d861f7 100644 --- a/api/v1beta1/ocicluster_conversion_test.go +++ b/api/v1beta1/ocicluster_conversion_test.go @@ -25,6 +25,83 @@ import ( "sigs.k8s.io/controller-runtime/pkg/conversion" ) +func TestOCICluster_BackendSetsRoundTrip(t *testing.T) { + t.Run("v1beta1 legacy -> v1beta2 -> v1beta1 keeps legacy shape", func(t *testing.T) { + src := &OCICluster{ + Spec: OCIClusterSpec{ + NetworkSpec: NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + }, + }, + } + + hub := &v1beta2.OCICluster{} + if err := src.ConvertTo(hub); err != nil { + t.Fatalf("convert to hub failed: %v", err) + } + if len(hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) != 0 { + t.Fatalf("expected no canonical backendSets to be synthesized during conversion, got %#v", hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) + } + if hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen == nil || !*hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen { + t.Fatalf("expected legacy backendSetDetails to be preserved on hub") + } + + restored := &OCICluster{} + if err := restored.ConvertFrom(hub); err != nil { + t.Fatalf("convert from hub failed: %v", err) + } + if restored.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen == nil || !*restored.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen { + t.Fatalf("expected legacy backendSetDetails after roundtrip, got %#v", restored.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails) + } + }) + + t.Run("v1beta2 canonical -> v1beta1 -> v1beta2 keeps canonical shape", func(t *testing.T) { + srcHub := &v1beta2.OCICluster{ + Spec: v1beta2.OCIClusterSpec{ + NetworkSpec: v1beta2.NetworkSpec{ + APIServerLB: v1beta2.LoadBalancer{ + NLBSpec: v1beta2.NLBSpec{ + BackendSets: []v1beta2.NLBBackendSet{ + { + Name: "new-set", + BackendSetDetails: v1beta2.BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + }, + }, + }, + }, + } + + spoke := &OCICluster{} + if err := spoke.ConvertFrom(srcHub); err != nil { + t.Fatalf("convert from hub failed: %v", err) + } + if len(spoke.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) != 1 || spoke.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].Name != "new-set" { + t.Fatalf("expected canonical backendSets after down-conversion, got %#v", spoke.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) + } + + restoredHub := &v1beta2.OCICluster{} + if err := spoke.ConvertTo(restoredHub); err != nil { + t.Fatalf("convert back to hub failed: %v", err) + } + if len(restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) != 1 || restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].Name != "new-set" { + t.Fatalf("expected canonical backendSets after hub roundtrip, got %#v", restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) + } + if restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].BackendSetDetails.IsFailOpen == nil || !*restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].BackendSetDetails.IsFailOpen { + t.Fatalf("expected backendSetDetails to remain intact after roundtrip") + } + }) +} + func TestOCICluster_ConvertTo(t *testing.T) { type fields struct { TypeMeta metav1.TypeMeta diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index af1ac3ad0..499dc899c 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -982,14 +982,43 @@ type LoadBalancer struct { // +optional LoadBalancerId *string `json:"loadBalancerId,omitempty"` - // The NLB Spec + // Shared API server load balancer settings used for both `loadBalancerType: nlb` and `loadBalancerType: lb`. + // The field name is historical; it is not limited to the NLB implementation. // +optional NLBSpec NLBSpec `json:"nlbSpec,omitempty"` } -// NLBSpec specifies the NLB spec. +// NewAPIServerLoadBalancer returns a LoadBalancer configured with the shared API server settings. +// The underlying `nlbSpec` field name is historical and retained for API compatibility. +func NewAPIServerLoadBalancer(name string, spec NLBSpec) LoadBalancer { + return LoadBalancer{ + Name: name, + NLBSpec: spec, + } +} + +// APIServerLoadBalancerSpec returns the shared API server load balancer settings. +func (in *LoadBalancer) APIServerLoadBalancerSpec() *NLBSpec { + return &in.NLBSpec +} + +// CanonicalAPIServerBackendSets returns the canonical API server backend set list for this load balancer. +func (in *LoadBalancer) CanonicalAPIServerBackendSets() []NLBBackendSet { + return in.APIServerLoadBalancerSpec().CanonicalBackendSets() +} + +// NLBSpec specifies the shared API server load balancer settings. +// The type name is historical and applies to both `loadBalancerType: nlb` and `loadBalancerType: lb`. type NLBSpec struct { + // BackendSets specifies the canonical list of API server backend sets. + // When set, this field takes precedence over the legacy `backendSetDetails` field. + // +optional + // +listType=map + // +listMapKey=name + BackendSets []NLBBackendSet `json:"backendSets,omitempty"` + // BackendSetDetails specifies the configuration of a network load balancer backend set. + // Deprecated: use `backendSets` instead. // +optional BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` @@ -1000,7 +1029,25 @@ type NLBSpec struct { ReservedIpIds []string `json:"reservedIpIds,omitempty"` } -// BackendSetDetails specifies the configuration of a network load balancer backend set. +// NLBBackendSet specifies the configuration for a named API server backend set. +// The type name is historical and applies to both load balancer implementations. +type NLBBackendSet struct { + // Name is the API server backend set identifier. + Name string `json:"name"` + + // ListenerPort is the load balancer listener port routed to this backend set. + // When omitted, the cluster API server port is used. Explicit values are restricted + // to the supported API server listener ports. + // +optional + ListenerPort *int32 `json:"listenerPort,omitempty"` + + // BackendSetDetails specifies the backend set behavior. + // +optional + BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` +} + +// BackendSetDetails specifies the configuration of an API server backend set. +// The fields cover shared behavior used by both load balancer implementations. type BackendSetDetails struct { // If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. // Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. @@ -1031,6 +1078,14 @@ type HealthChecker struct { UrlPath *string `json:"urlPath,omitempty"` } +// CanonicalBackendSets returns the canonical API server backend set list. +// Precedence: +// 1. If `backendSets` is configured, it is used as-is. +// 2. Otherwise, a single backend set named APIServerLBBackendSetName is synthesized from the legacy `backendSetDetails`. +func (in *NLBSpec) CanonicalBackendSets() []NLBBackendSet { + return canonicalAPIServerBackendSets(in.BackendSets, in.BackendSetDetails) +} + // NetworkSpec specifies what the OCI networking resources should look like. type NetworkSpec struct { // SkipNetworkManagement defines if the networking spec(VCN related) specified by the user needs to be reconciled(actioned-upon) diff --git a/api/v1beta1/types_test.go b/api/v1beta1/types_test.go new file mode 100644 index 000000000..39dd18733 --- /dev/null +++ b/api/v1beta1/types_test.go @@ -0,0 +1,92 @@ +package v1beta1 + +import ( + "encoding/json" + "testing" +) + +func TestNLBSpecCanonicalBackendSets_Precedence(t *testing.T) { + t.Run("uses backendSets when present", func(t *testing.T) { + spec := NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "new-set"}, + }, + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 || got[0].Name != "new-set" { + t.Fatalf("expected canonical backendSets to be used, got %#v", got) + } + }) + + t.Run("falls back to legacy backendSetDetails", func(t *testing.T) { + spec := NLBSpec{ + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 { + t.Fatalf("expected one synthesized backend set, got %#v", got) + } + if got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected synthesized backend set name %q, got %q", APIServerLBBackendSetName, got[0].Name) + } + if got[0].BackendSetDetails.IsFailOpen == nil || *got[0].BackendSetDetails.IsFailOpen != true { + t.Fatalf("expected legacy backend set details to be preserved, got %#v", got[0].BackendSetDetails) + } + }) + + t.Run("defaults to legacy backend set name when empty", func(t *testing.T) { + spec := NLBSpec{} + + got := spec.CanonicalBackendSets() + if len(got) != 1 { + t.Fatalf("expected one synthesized backend set, got %#v", got) + } + if got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected synthesized backend set name %q, got %q", APIServerLBBackendSetName, got[0].Name) + } + }) +} + +func TestNLBSpecLegacyDecode_ToCanonical(t *testing.T) { + raw := []byte(`{"backendSetDetails":{"isFailOpen":true}}`) + var spec NLBSpec + if err := json.Unmarshal(raw, &spec); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 || got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected legacy decode to map to one canonical backend set, got %#v", got) + } +} + +func TestNLBSpecDeepCopy_NoSharedPointers(t *testing.T) { + spec := &NLBSpec{ + BackendSets: []NLBBackendSet{ + { + Name: "set-a", + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + } + + cp := spec.DeepCopy() + *cp.BackendSets[0].BackendSetDetails.IsFailOpen = false + + if spec.BackendSets[0].BackendSetDetails.IsFailOpen == nil || *spec.BackendSets[0].BackendSetDetails.IsFailOpen != true { + t.Fatalf("expected deepcopy to isolate nested pointers, original=%#v", spec.BackendSets[0].BackendSetDetails) + } +} + +func boolPtr(v bool) *bool { + return &v +} diff --git a/api/v1beta1/validator.go b/api/v1beta1/validator.go index a1e136f13..8076f6be8 100644 --- a/api/v1beta1/validator.go +++ b/api/v1beta1/validator.go @@ -32,6 +32,9 @@ const ( // can't use: \/"'[]:|<>+=;,.?*@&, Can't start with underscore. Can't end with period or hyphen. // not using . in the name to avoid issues when the name is part of DNS name. clusterNameRegex = `^[a-z0-9][a-z0-9-]{0,42}[a-z0-9]$` + + apiServerBackendSetNameRegex = `^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$` + supportedSecondaryListenerPort int32 = 9345 ) // invalidNameRegex is a broad regex used to validate allows names in OCI @@ -69,7 +72,7 @@ func ValidRegion(stringRegion string) bool { } // ValidateNetworkSpec validates the NetworkSpec -func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old NetworkSpec, fldPath *field.Path) field.ErrorList { +func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old NetworkSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList if len(networkSpec.Vcn.CIDR) > 0 { @@ -84,12 +87,18 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network allErrs = append(allErrs, validateNSGs(validRoles, networkSpec.Vcn.NetworkSecurityGroups, fldPath.Child("networkSecurityGroups"))...) } + allErrs = append(allErrs, validateAPIServerLBBackendSets(networkSpec.APIServerLB.NLBSpec, apiServerPort, fldPath.Child("apiServerLoadBalancer").Child("nlbSpec"))...) + if len(allErrs) == 0 { return nil } return allErrs } +func validateAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { + return validateSharedAPIServerLBBackendSets(spec, apiServerPort, fldPath) +} + // validateVCNCIDR validates the CIDR of a VNC. func validateVCNCIDR(vncCIDR string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta1/validator_backendsets_test.go b/api/v1beta1/validator_backendsets_test.go new file mode 100644 index 000000000..1b6fea0ad --- /dev/null +++ b/api/v1beta1/validator_backendsets_test.go @@ -0,0 +1,304 @@ +package v1beta1 + +import ( + "strings" + "testing" + + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func TestValidateNetworkSpec_BackendSets(t *testing.T) { + t.Run("rejects mixed legacy and canonical fields", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "new-set"}}, + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for mixed fields") + } + if !strings.Contains(errs[0].Error(), "backendSetDetails") { + t.Fatalf("expected backendSetDetails path in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects duplicate backend set names", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "dup"}, {Name: "dup"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate names") + } + if !strings.Contains(errs[0].Error(), "duplicate backend set name") { + t.Fatalf("expected duplicate-name guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects invalid backend set name format", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "bad name"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for invalid name format") + } + if !strings.Contains(errs[0].Error(), "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$") { + t.Fatalf("expected regex guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts canonical backend sets with distinct effective ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "apiserver_a"}, {Name: "apiserver-b", ListenerPort: int32Ptr(9345)}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("rejects duplicate listener ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a", ListenerPort: int32Ptr(9345)}, + {Name: "set-b", ListenerPort: int32Ptr(9345)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate listener ports") + } + if !strings.Contains(errs[0].Error(), "duplicate effective listenerPort 9345") { + t.Fatalf("expected duplicate effective listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects unsupported listener ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a", ListenerPort: int32Ptr(10250)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for unsupported listener port") + } + if !strings.Contains(errs[0].Error(), "must be one of 6443, 9345") { + t.Fatalf("expected supported-port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects multiple omitted listener ports even when cluster API server port is unknown", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b"}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for multiple omitted listener ports") + } + if !strings.Contains(errs[0].Error(), "multiple backend sets omit listenerPort") { + t.Fatalf("expected omitted listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts omitted and explicit listener ports when cluster API server port is unknown", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(6443)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("rejects omitted and explicit listener ports when cluster API server port is provided", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(7443)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate effective listener ports") + } + if !strings.Contains(errs[0].Error(), "duplicate effective listenerPort 7443") { + t.Fatalf("expected duplicate effective listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts explicit listener port that differs from provided cluster API server port", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(6443)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("rejects unsupported listener port when cluster API server port is provided", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(10250)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for unsupported listener port") + } + if !strings.Contains(errs[0].Error(), "must be one of 6443, 7443, 9345") { + t.Fatalf("expected supported-port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts single omitted listener port", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "set-a"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) +} + +func int32Ptr(v int32) *int32 { + return &v +} diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 279055d0f..9770a853f 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -370,6 +370,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*NLBBackendSet)(nil), (*v1beta2.NLBBackendSet)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(a.(*NLBBackendSet), b.(*v1beta2.NLBBackendSet), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.NLBBackendSet)(nil), (*NLBBackendSet)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet(a.(*v1beta2.NLBBackendSet), b.(*NLBBackendSet), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*NLBSpec)(nil), (*v1beta2.NLBSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_NLBSpec_To_v1beta2_NLBSpec(a.(*NLBSpec), b.(*v1beta2.NLBSpec), scope) }); err != nil { @@ -1860,7 +1870,36 @@ func autoConvert_v1beta2_LoadBalancer_To_v1beta1_LoadBalancer(in *v1beta2.LoadBa return nil } +func autoConvert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(in *NLBBackendSet, out *v1beta2.NLBBackendSet, s conversion.Scope) error { + out.Name = in.Name + out.ListenerPort = (*int32)(unsafe.Pointer(in.ListenerPort)) + if err := Convert_v1beta1_BackendSetDetails_To_v1beta2_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet is an autogenerated conversion function. +func Convert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(in *NLBBackendSet, out *v1beta2.NLBBackendSet, s conversion.Scope) error { + return autoConvert_v1beta1_NLBBackendSet_To_v1beta2_NLBBackendSet(in, out, s) +} + +func autoConvert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet(in *v1beta2.NLBBackendSet, out *NLBBackendSet, s conversion.Scope) error { + out.Name = in.Name + out.ListenerPort = (*int32)(unsafe.Pointer(in.ListenerPort)) + if err := Convert_v1beta2_BackendSetDetails_To_v1beta1_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet is an autogenerated conversion function. +func Convert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet(in *v1beta2.NLBBackendSet, out *NLBBackendSet, s conversion.Scope) error { + return autoConvert_v1beta2_NLBBackendSet_To_v1beta1_NLBBackendSet(in, out, s) +} + func autoConvert_v1beta1_NLBSpec_To_v1beta2_NLBSpec(in *NLBSpec, out *v1beta2.NLBSpec, s conversion.Scope) error { + out.BackendSets = *(*[]v1beta2.NLBBackendSet)(unsafe.Pointer(&in.BackendSets)) if err := Convert_v1beta1_BackendSetDetails_To_v1beta2_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { return err } @@ -1874,6 +1913,7 @@ func Convert_v1beta1_NLBSpec_To_v1beta2_NLBSpec(in *NLBSpec, out *v1beta2.NLBSpe } func autoConvert_v1beta2_NLBSpec_To_v1beta1_NLBSpec(in *v1beta2.NLBSpec, out *NLBSpec, s conversion.Scope) error { + out.BackendSets = *(*[]NLBBackendSet)(unsafe.Pointer(&in.BackendSets)) if err := Convert_v1beta2_BackendSetDetails_To_v1beta1_BackendSetDetails(&in.BackendSetDetails, &out.BackendSetDetails, s); err != nil { return err } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0ea122fc6..60aef0fc5 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1105,9 +1105,37 @@ func (in *LoadBalancer) DeepCopy() *LoadBalancer { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NLBBackendSet) DeepCopyInto(out *NLBBackendSet) { + *out = *in + if in.ListenerPort != nil { + in, out := &in.ListenerPort, &out.ListenerPort + *out = new(int32) + **out = **in + } + in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NLBBackendSet. +func (in *NLBBackendSet) DeepCopy() *NLBBackendSet { + if in == nil { + return nil + } + out := new(NLBBackendSet) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NLBSpec) DeepCopyInto(out *NLBSpec) { *out = *in + if in.BackendSets != nil { + in, out := &in.BackendSets, &out.BackendSets + *out = make([]NLBBackendSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) if in.ReservedIpIds != nil { in, out := &in.ReservedIpIds, &out.ReservedIpIds diff --git a/api/v1beta2/backendsets_shared.go b/api/v1beta2/backendsets_shared.go new file mode 100644 index 000000000..abd5bf4bb --- /dev/null +++ b/api/v1beta2/backendsets_shared.go @@ -0,0 +1,42 @@ +package v1beta2 + +import ( + apiserverlb "github.com/oracle/cluster-api-provider-oci/api/internal/apiserverlb" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func canonicalAPIServerBackendSets(configured []NLBBackendSet, legacy BackendSetDetails) []NLBBackendSet { + return apiserverlb.CanonicalBackendSets(configured, func() NLBBackendSet { + return NLBBackendSet{ + Name: APIServerLBBackendSetName, + BackendSetDetails: legacy, + } + }) +} + +func validateSharedAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { + backendSets := make([]apiserverlb.BackendSet, 0, len(spec.BackendSets)) + for _, backendSet := range spec.BackendSets { + backendSets = append(backendSets, apiserverlb.BackendSet{ + Name: backendSet.Name, + ListenerPort: backendSet.ListenerPort, + }) + } + return apiserverlb.ValidateBackendSets(backendSets, spec.BackendSetDetails != (BackendSetDetails{}), apiServerPort, supportedSecondaryListenerPort, fldPath) +} + +func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) { + return apiserverlb.EffectiveAPIServerListenerPort(listenerPort, apiServerPort) +} + +func isSupportedAPIServerListenerPort(port int32, apiServerPort *int32) bool { + return apiserverlb.IsSupportedAPIServerListenerPort(port, apiServerPort, supportedSecondaryListenerPort) +} + +func supportedAPIServerListenerPorts(apiServerPort *int32) []int32 { + return apiserverlb.SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort) +} + +func supportedAPIServerListenerPortsString(apiServerPort *int32) string { + return apiserverlb.SupportedAPIServerListenerPortsString(apiServerPort, supportedSecondaryListenerPort) +} diff --git a/api/v1beta2/cluster_apiserver_port.go b/api/v1beta2/cluster_apiserver_port.go new file mode 100644 index 000000000..79881bc94 --- /dev/null +++ b/api/v1beta2/cluster_apiserver_port.go @@ -0,0 +1,54 @@ +package v1beta2 + +import ( + "context" + + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + pkgerrors "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func lookupClusterAPIServerPort(ctx context.Context, c client.Client, obj metav1.ObjectMeta) (*int32, error) { + if c == nil { + return nil, nil + } + + cluster, err := util.GetOwnerCluster(ctx, c, obj) + if err != nil { + return nil, err + } + if cluster == nil { + cluster, err = util.GetClusterFromMetadata(ctx, c, obj) + if err != nil { + if pkgerrors.Cause(err) == util.ErrNoCluster { + return nil, nil + } + return nil, err + } + } + if cluster == nil { + return nil, nil + } + + port := cluster.Spec.ClusterNetwork.APIServerPort + if port == 0 { + port = ociutil.DefaultAPIServerPort + } + return &port, nil +} + +func newClusterOwnerReference(cluster *clusterv1.Cluster) metav1.OwnerReference { + controller := true + blockOwnerDeletion := true + return metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + Controller: &controller, + BlockOwnerDeletion: &blockOwnerDeletion, + } +} diff --git a/api/v1beta2/cluster_apiserver_port_test.go b/api/v1beta2/cluster_apiserver_port_test.go new file mode 100644 index 000000000..88373239c --- /dev/null +++ b/api/v1beta2/cluster_apiserver_port_test.go @@ -0,0 +1,77 @@ +package v1beta2 + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + ctrlclientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestLookupClusterAPIServerPort(t *testing.T) { + t.Run("uses owner cluster api server port when set", func(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add cluster api scheme: %v", err) + } + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + UID: "cluster-uid", + }, + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: clusterv1.ClusterNetwork{APIServerPort: 7443}, + }, + } + + client := ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster).Build() + objMeta := metav1.ObjectMeta{ + Name: "test-oci-cluster", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{newClusterOwnerReference(cluster)}, + } + + port, err := lookupClusterAPIServerPort(context.Background(), client, objMeta) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if port == nil || *port != 7443 { + t.Fatalf("expected api server port 7443, got %v", port) + } + }) + + t.Run("falls back to cluster-name label and default port", func(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add cluster api scheme: %v", err) + } + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + } + + client := ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster).Build() + objMeta := metav1.ObjectMeta{ + Name: "test-oci-cluster", + Namespace: "default", + Labels: map[string]string{ + clusterv1.ClusterNameLabel: cluster.Name, + }, + } + + port, err := lookupClusterAPIServerPort(context.Background(), client, objMeta) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if port == nil || *port != 6443 { + t.Fatalf("expected api server port 6443, got %v", port) + } + }) +} diff --git a/api/v1beta2/ocicluster_webhook.go b/api/v1beta2/ocicluster_webhook.go index 17c614281..412438130 100644 --- a/api/v1beta2/ocicluster_webhook.go +++ b/api/v1beta2/ocicluster_webhook.go @@ -29,13 +29,17 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) var clusterlogger = ctrl.Log.WithName("ocicluster-resource") -type OCIClusterWebhook struct{} +// +kubebuilder:object:generate=false +type OCIClusterWebhook struct { + Client client.Client +} var ( _ webhook.CustomDefaulter = &OCIClusterWebhook{} @@ -63,7 +67,7 @@ func (*OCIClusterWebhook) Default(_ context.Context, obj runtime.Object) error { } func (c *OCICluster) SetupWebhookWithManager(mgr ctrl.Manager) error { - w := new(OCIClusterWebhook) + w := &OCIClusterWebhook{Client: mgr.GetClient()} return ctrl.NewWebhookManagedBy(mgr). For(c). WithDefaulter(w). @@ -72,7 +76,7 @@ func (c *OCICluster) SetupWebhookWithManager(mgr ctrl.Manager) error { } // ValidateCreate implements webhook.Validator so a webhook will be registered for the type. -func (*OCIClusterWebhook) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { +func (w *OCIClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { c, ok := obj.(*OCICluster) if !ok { return nil, fmt.Errorf("expected an OCICluster object but got %T", c) @@ -80,6 +84,11 @@ func (*OCIClusterWebhook) ValidateCreate(_ context.Context, obj runtime.Object) clusterlogger.Info("validate update cluster", "name", c.Name) + apiServerPort, err := lookupClusterAPIServerPort(ctx, w.Client, c.ObjectMeta) + if err != nil { + return nil, err + } + var allErrs field.ErrorList var ipv6hextets []*string var hextatassigned bool @@ -163,7 +172,7 @@ func (*OCIClusterWebhook) ValidateCreate(_ context.Context, obj runtime.Object) } } - allErrs = append(allErrs, c.validate(nil)...) + allErrs = append(allErrs, c.validate(nil, apiServerPort)...) if len(allErrs) == 0 { return nil, nil @@ -184,7 +193,7 @@ func (*OCIClusterWebhook) ValidateDelete(_ context.Context, obj runtime.Object) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. -func (*OCIClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runtime.Object) (admission.Warnings, error) { +func (w *OCIClusterWebhook) ValidateUpdate(ctx context.Context, oldRaw, newObj runtime.Object) (admission.Warnings, error) { c, ok := newObj.(*OCICluster) if !ok { return nil, fmt.Errorf("expected an OCICluster object but got %T", c) @@ -192,6 +201,11 @@ func (*OCIClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runti clusterlogger.Info("validate update cluster", "name", c.Name) + apiServerPort, err := lookupClusterAPIServerPort(ctx, w.Client, c.ObjectMeta) + if err != nil { + return nil, err + } + var allErrs field.ErrorList oldCluster, ok := oldRaw.(*OCICluster) @@ -262,7 +276,7 @@ func (*OCIClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runti } } - allErrs = append(allErrs, c.validate(oldCluster)...) + allErrs = append(allErrs, c.validate(oldCluster, apiServerPort)...) if len(allErrs) == 0 { return nil, nil @@ -271,7 +285,7 @@ func (*OCIClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runti return nil, apierrors.NewInvalid(c.GroupVersionKind().GroupKind(), c.Name, allErrs) } -func (c *OCICluster) validate(old *OCICluster) field.ErrorList { +func (c *OCICluster) validate(old *OCICluster, apiServerPort *int32) field.ErrorList { var allErrs field.ErrorList var oldNetworkSpec NetworkSpec @@ -279,7 +293,7 @@ func (c *OCICluster) validate(old *OCICluster) field.ErrorList { oldNetworkSpec = old.Spec.NetworkSpec } - allErrs = append(allErrs, ValidateNetworkSpec(OCIClusterSubnetRoles, c.Spec.NetworkSpec, oldNetworkSpec, field.NewPath("spec").Child("networkSpec"))...) + allErrs = append(allErrs, ValidateNetworkSpec(OCIClusterSubnetRoles, c.Spec.NetworkSpec, oldNetworkSpec, apiServerPort, field.NewPath("spec").Child("networkSpec"))...) allErrs = append(allErrs, ValidateClusterName(c.Name)...) if len(c.Spec.CompartmentId) <= 0 { diff --git a/api/v1beta2/ocimanagedcluster_webhook.go b/api/v1beta2/ocimanagedcluster_webhook.go index 5a4e67771..38dc829db 100644 --- a/api/v1beta2/ocimanagedcluster_webhook.go +++ b/api/v1beta2/ocimanagedcluster_webhook.go @@ -27,13 +27,17 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) var managedclusterlogger = ctrl.Log.WithName("ocimanagedcluster-resource") -type OCIManagedClusterWebhook struct{} +// +kubebuilder:object:generate=false +type OCIManagedClusterWebhook struct { + Client client.Client +} var ( _ webhook.CustomDefaulter = &OCIManagedClusterWebhook{} @@ -119,7 +123,7 @@ func (*OCIManagedClusterWebhook) Default(_ context.Context, obj runtime.Object) } func (c *OCIManagedCluster) SetupWebhookWithManager(mgr ctrl.Manager) error { - w := new(OCIManagedClusterWebhook) + w := &OCIManagedClusterWebhook{Client: mgr.GetClient()} return ctrl.NewWebhookManagedBy(mgr). For(c). WithDefaulter(w). @@ -128,16 +132,21 @@ func (c *OCIManagedCluster) SetupWebhookWithManager(mgr ctrl.Manager) error { } // ValidateCreate implements webhook.Validator so a webhook will be registered for the type. -func (*OCIManagedClusterWebhook) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { +func (w *OCIManagedClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { c, ok := obj.(*OCIManagedCluster) if !ok { return nil, fmt.Errorf("expected an OCIManagedCluster object but got %T", c) } managedclusterlogger.Info("validate create cluster", "name", c.Name) + apiServerPort, err := lookupClusterAPIServerPort(ctx, w.Client, c.ObjectMeta) + if err != nil { + return nil, err + } + var allErrs field.ErrorList - allErrs = append(allErrs, c.validate(nil)...) + allErrs = append(allErrs, c.validate(nil, apiServerPort)...) if len(allErrs) == 0 { return nil, nil @@ -159,7 +168,7 @@ func (*OCIManagedClusterWebhook) ValidateDelete(_ context.Context, obj runtime.O } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. -func (*OCIManagedClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newObj runtime.Object) (admission.Warnings, error) { +func (w *OCIManagedClusterWebhook) ValidateUpdate(ctx context.Context, oldRaw, newObj runtime.Object) (admission.Warnings, error) { c, ok := newObj.(*OCIManagedCluster) if !ok { return nil, fmt.Errorf("expected an OCIManagedCluster object but got %T", c) @@ -186,7 +195,12 @@ func (*OCIManagedClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newOb allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "compartmentId"), c.Spec.CompartmentId, "field is immutable")) } - allErrs = append(allErrs, c.validate(oldCluster)...) + apiServerPort, err := lookupClusterAPIServerPort(ctx, w.Client, c.ObjectMeta) + if err != nil { + return nil, err + } + + allErrs = append(allErrs, c.validate(oldCluster, apiServerPort)...) if len(allErrs) == 0 { return nil, nil @@ -195,7 +209,7 @@ func (*OCIManagedClusterWebhook) ValidateUpdate(_ context.Context, oldRaw, newOb return nil, apierrors.NewInvalid(c.GroupVersionKind().GroupKind(), c.Name, allErrs) } -func (c *OCIManagedCluster) validate(old *OCIManagedCluster) field.ErrorList { +func (c *OCIManagedCluster) validate(old *OCIManagedCluster, apiServerPort *int32) field.ErrorList { var allErrs field.ErrorList var oldNetworkSpec NetworkSpec @@ -203,7 +217,7 @@ func (c *OCIManagedCluster) validate(old *OCIManagedCluster) field.ErrorList { oldNetworkSpec = old.Spec.NetworkSpec } - allErrs = append(allErrs, ValidateNetworkSpec(OCIManagedClusterSubnetRoles, c.Spec.NetworkSpec, oldNetworkSpec, field.NewPath("spec").Child("networkSpec"))...) + allErrs = append(allErrs, ValidateNetworkSpec(OCIManagedClusterSubnetRoles, c.Spec.NetworkSpec, oldNetworkSpec, apiServerPort, field.NewPath("spec").Child("networkSpec"))...) allErrs = append(allErrs, ValidateClusterName(c.Name)...) if len(c.Spec.CompartmentId) <= 0 { diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index 532ff82e9..54f83cc5f 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -991,14 +991,43 @@ type LoadBalancer struct { // +optional LoadBalancerType LoadBalancerType `json:"loadBalancerType,omitempty"` - // The NLB Spec + // Shared API server load balancer settings used for both `loadBalancerType: nlb` and `loadBalancerType: lb`. + // The field name is historical; it is not limited to the NLB implementation. // +optional NLBSpec NLBSpec `json:"nlbSpec,omitempty"` } -// NLBSpec specifies the NLB spec. +// NewAPIServerLoadBalancer returns a LoadBalancer configured with the shared API server settings. +// The underlying `nlbSpec` field name is historical and retained for API compatibility. +func NewAPIServerLoadBalancer(name string, spec NLBSpec) LoadBalancer { + return LoadBalancer{ + Name: name, + NLBSpec: spec, + } +} + +// APIServerLoadBalancerSpec returns the shared API server load balancer settings. +func (in *LoadBalancer) APIServerLoadBalancerSpec() *NLBSpec { + return &in.NLBSpec +} + +// CanonicalAPIServerBackendSets returns the canonical API server backend set list for this load balancer. +func (in *LoadBalancer) CanonicalAPIServerBackendSets() []NLBBackendSet { + return in.APIServerLoadBalancerSpec().CanonicalBackendSets() +} + +// NLBSpec specifies the shared API server load balancer settings. +// The type name is historical and applies to both `loadBalancerType: nlb` and `loadBalancerType: lb`. type NLBSpec struct { + // BackendSets specifies the canonical list of API server backend sets. + // When set, this field takes precedence over the legacy `backendSetDetails` field. + // +optional + // +listType=map + // +listMapKey=name + BackendSets []NLBBackendSet `json:"backendSets,omitempty"` + // BackendSetDetails specifies the configuration of a network load balancer backend set. + // Deprecated: use `backendSets` instead. // +optional BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` @@ -1009,7 +1038,25 @@ type NLBSpec struct { ReservedIpIds []string `json:"reservedIpIds,omitempty"` } -// BackendSetDetails specifies the configuration of a network load balancer backend set. +// NLBBackendSet specifies the configuration for a named API server backend set. +// The type name is historical and applies to both load balancer implementations. +type NLBBackendSet struct { + // Name is the API server backend set identifier. + Name string `json:"name"` + + // ListenerPort is the load balancer listener port routed to this backend set. + // When omitted, the cluster API server port is used. Explicit values are restricted + // to the supported API server listener ports. + // +optional + ListenerPort *int32 `json:"listenerPort,omitempty"` + + // BackendSetDetails specifies the backend set behavior. + // +optional + BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"` +} + +// BackendSetDetails specifies the configuration of an API server backend set. +// The fields cover shared behavior used by both load balancer implementations. type BackendSetDetails struct { // If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. // Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. @@ -1041,6 +1088,14 @@ type HealthChecker struct { UrlPath *string `json:"urlPath,omitempty"` } +// CanonicalBackendSets returns the canonical API server backend set list. +// Precedence: +// 1. If `backendSets` is configured, it is used as-is. +// 2. Otherwise, a single backend set named APIServerLBBackendSetName is synthesized from the legacy `backendSetDetails`. +func (in *NLBSpec) CanonicalBackendSets() []NLBBackendSet { + return canonicalAPIServerBackendSets(in.BackendSets, in.BackendSetDetails) +} + // NetworkSpec specifies what the OCI networking resources should look like. type NetworkSpec struct { // SkipNetworkManagement defines if the networking spec(VCN related) specified by the user needs to be reconciled(actioned-upon) diff --git a/api/v1beta2/types_test.go b/api/v1beta2/types_test.go new file mode 100644 index 000000000..4c73e692a --- /dev/null +++ b/api/v1beta2/types_test.go @@ -0,0 +1,92 @@ +package v1beta2 + +import ( + "encoding/json" + "testing" +) + +func TestNLBSpecCanonicalBackendSets_Precedence(t *testing.T) { + t.Run("uses backendSets when present", func(t *testing.T) { + spec := NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "new-set"}, + }, + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 || got[0].Name != "new-set" { + t.Fatalf("expected canonical backendSets to be used, got %#v", got) + } + }) + + t.Run("falls back to legacy backendSetDetails", func(t *testing.T) { + spec := NLBSpec{ + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 { + t.Fatalf("expected one synthesized backend set, got %#v", got) + } + if got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected synthesized backend set name %q, got %q", APIServerLBBackendSetName, got[0].Name) + } + if got[0].BackendSetDetails.IsFailOpen == nil || *got[0].BackendSetDetails.IsFailOpen != true { + t.Fatalf("expected legacy backend set details to be preserved, got %#v", got[0].BackendSetDetails) + } + }) + + t.Run("defaults to legacy backend set name when empty", func(t *testing.T) { + spec := NLBSpec{} + + got := spec.CanonicalBackendSets() + if len(got) != 1 { + t.Fatalf("expected one synthesized backend set, got %#v", got) + } + if got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected synthesized backend set name %q, got %q", APIServerLBBackendSetName, got[0].Name) + } + }) +} + +func TestNLBSpecLegacyDecode_ToCanonical(t *testing.T) { + raw := []byte(`{"backendSetDetails":{"isFailOpen":true}}`) + var spec NLBSpec + if err := json.Unmarshal(raw, &spec); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + + got := spec.CanonicalBackendSets() + if len(got) != 1 || got[0].Name != APIServerLBBackendSetName { + t.Fatalf("expected legacy decode to map to one canonical backend set, got %#v", got) + } +} + +func TestNLBSpecDeepCopy_NoSharedPointers(t *testing.T) { + spec := &NLBSpec{ + BackendSets: []NLBBackendSet{ + { + Name: "set-a", + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + } + + cp := spec.DeepCopy() + *cp.BackendSets[0].BackendSetDetails.IsFailOpen = false + + if spec.BackendSets[0].BackendSetDetails.IsFailOpen == nil || *spec.BackendSets[0].BackendSetDetails.IsFailOpen != true { + t.Fatalf("expected deepcopy to isolate nested pointers, original=%#v", spec.BackendSets[0].BackendSetDetails) + } +} + +func boolPtr(v bool) *bool { + return &v +} diff --git a/api/v1beta2/validator.go b/api/v1beta2/validator.go index 97854df10..f48d4c256 100644 --- a/api/v1beta2/validator.go +++ b/api/v1beta2/validator.go @@ -38,19 +38,21 @@ const ( ingressRulesType = "ingressRules" // Error message formats for NSG security rule validation - udpDestinationPortRangeMaxRequiredFormat = "invalid %s: UdpOptions DestinationPortRange Max may not be empty" - udpDestinationPortRangeMinRequiredFormat = "invalid %s: UdpOptions DestinationPortRange Min may not be empty" - udpSourcePortRangeMaxRequiredFormat = "invalid %s: UdpOptions SourcePortRange Max may not be empty" - udpSourcePortRangeMinRequiredFormat = "invalid %s: UdpOptions SourcePortRange Min may not be empty" - tcpDestinationPortRangeMaxRequiredFormat = "invalid %s: TcpOptions DestinationPortRange Max may not be empty" - tcpDestinationPortRangeMinRequiredFormat = "invalid %s: TcpOptions DestinationPortRange Min may not be empty" - tcpSourcePortRangeMaxRequiredFormat = "invalid %s: TcpOptions SourcePortRange Max may not be empty" - tcpSourcePortRangeMinRequiredFormat = "invalid %s: TcpOptions SourcePortRange Min may not be empty" - icmpTypeRequiredFormat = "invalid %s: IcmpOptions Type may not be empty" - destinationRequiredFormat = "invalid %s: Destination may not be empty" - sourceRequiredFormat = "invalid %s: Source may not be empty" - protocolRequiredFormat = "invalid %s: Protocol may not be empty" - invalidCIDRFormatFormat = "invalid %s: CIDR format" + udpDestinationPortRangeMaxRequiredFormat = "invalid %s: UdpOptions DestinationPortRange Max may not be empty" + udpDestinationPortRangeMinRequiredFormat = "invalid %s: UdpOptions DestinationPortRange Min may not be empty" + udpSourcePortRangeMaxRequiredFormat = "invalid %s: UdpOptions SourcePortRange Max may not be empty" + udpSourcePortRangeMinRequiredFormat = "invalid %s: UdpOptions SourcePortRange Min may not be empty" + tcpDestinationPortRangeMaxRequiredFormat = "invalid %s: TcpOptions DestinationPortRange Max may not be empty" + tcpDestinationPortRangeMinRequiredFormat = "invalid %s: TcpOptions DestinationPortRange Min may not be empty" + tcpSourcePortRangeMaxRequiredFormat = "invalid %s: TcpOptions SourcePortRange Max may not be empty" + tcpSourcePortRangeMinRequiredFormat = "invalid %s: TcpOptions SourcePortRange Min may not be empty" + icmpTypeRequiredFormat = "invalid %s: IcmpOptions Type may not be empty" + destinationRequiredFormat = "invalid %s: Destination may not be empty" + sourceRequiredFormat = "invalid %s: Source may not be empty" + protocolRequiredFormat = "invalid %s: Protocol may not be empty" + invalidCIDRFormatFormat = "invalid %s: CIDR format" + apiServerBackendSetNameRegex = `^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$` + supportedSecondaryListenerPort int32 = 9345 ) // invalidNameRegex is a broad regex used to validate allows names in OCI @@ -109,7 +111,7 @@ func ValidRegion(stringRegion string) bool { } // ValidateNetworkSpec validates the NetworkSpec -func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old NetworkSpec, fldPath *field.Path) field.ErrorList { +func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old NetworkSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList if len(networkSpec.Vcn.CIDR) > 0 { @@ -126,6 +128,7 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network nsgErrors := validateNSGs(validRoles, networkSpec.Vcn.NetworkSecurityGroup.List, fldPath.Child("networkSecurityGroups")) allErrs = append(allErrs, nsgErrors...) } + allErrs = append(allErrs, validateAPIServerLBBackendSets(networkSpec.APIServerLB.NLBSpec, apiServerPort, fldPath.Child("apiServerLoadBalancer").Child("nlbSpec"))...) if len(allErrs) == 0 { return nil @@ -133,6 +136,10 @@ func ValidateNetworkSpec(validRoles []Role, networkSpec NetworkSpec, old Network return allErrs } +func validateAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList { + return validateSharedAPIServerLBBackendSets(spec, apiServerPort, fldPath) +} + // validateVCNCIDR validates the CIDR of a VNC. func validateVCNCIDR(vncCIDR string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/api/v1beta2/validator_backendsets_test.go b/api/v1beta2/validator_backendsets_test.go new file mode 100644 index 000000000..952932394 --- /dev/null +++ b/api/v1beta2/validator_backendsets_test.go @@ -0,0 +1,304 @@ +package v1beta2 + +import ( + "strings" + "testing" + + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func TestValidateNetworkSpec_BackendSets(t *testing.T) { + t.Run("rejects mixed legacy and canonical fields", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "new-set"}}, + BackendSetDetails: BackendSetDetails{ + IsFailOpen: boolPtr(true), + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for mixed fields") + } + if !strings.Contains(errs[0].Error(), "backendSetDetails") { + t.Fatalf("expected backendSetDetails path in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects duplicate backend set names", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "dup"}, {Name: "dup"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate names") + } + if !strings.Contains(errs[0].Error(), "duplicate backend set name") { + t.Fatalf("expected duplicate-name guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects invalid backend set name format", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "bad name"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for invalid name format") + } + if !strings.Contains(errs[0].Error(), "must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$") { + t.Fatalf("expected regex guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts canonical backend sets with distinct effective ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "apiserver_a"}, {Name: "apiserver-b", ListenerPort: int32Ptr(9345)}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("rejects duplicate listener ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a", ListenerPort: int32Ptr(9345)}, + {Name: "set-b", ListenerPort: int32Ptr(9345)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate listener ports") + } + if !strings.Contains(errs[0].Error(), "duplicate effective listenerPort 9345") { + t.Fatalf("expected duplicate effective listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects unsupported listener ports", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a", ListenerPort: int32Ptr(10250)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for unsupported listener port") + } + if !strings.Contains(errs[0].Error(), "must be one of 6443, 9345") { + t.Fatalf("expected supported-port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("rejects multiple omitted listener ports even when cluster API server port is unknown", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b"}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for multiple omitted listener ports") + } + if !strings.Contains(errs[0].Error(), "multiple backend sets omit listenerPort") { + t.Fatalf("expected omitted listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts omitted and explicit listener ports when cluster API server port is unknown", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(6443)}, + }, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("rejects omitted and explicit listener ports when cluster API server port is provided", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(7443)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for duplicate effective listener ports") + } + if !strings.Contains(errs[0].Error(), "duplicate effective listenerPort 7443") { + t.Fatalf("expected duplicate effective listener port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts explicit listener port that differs from provided cluster API server port", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(6443)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) + + t.Run("rejects unsupported listener port when cluster API server port is provided", func(t *testing.T) { + apiServerPort := int32(7443) + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b", ListenerPort: int32Ptr(10250)}, + }, + }, + }, + }, + NetworkSpec{}, + &apiServerPort, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) == 0 { + t.Fatalf("expected validation error for unsupported listener port") + } + if !strings.Contains(errs[0].Error(), "must be one of 6443, 7443, 9345") { + t.Fatalf("expected supported-port guidance in error, got %q", errs[0].Error()) + } + }) + + t.Run("accepts single omitted listener port", func(t *testing.T) { + errs := ValidateNetworkSpec( + OCIClusterSubnetRoles, + NetworkSpec{ + APIServerLB: LoadBalancer{ + NLBSpec: NLBSpec{ + BackendSets: []NLBBackendSet{{Name: "set-a"}}, + }, + }, + }, + NetworkSpec{}, + nil, + field.NewPath("spec").Child("networkSpec"), + ) + + if len(errs) != 0 { + t.Fatalf("expected no validation errors, got %v", errs.ToAggregate()) + } + }) +} + +func int32Ptr(v int32) *int32 { + return &v +} diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 4e4150c08..a91fa83d7 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -1347,9 +1347,37 @@ func (in *NATGateway) DeepCopy() *NATGateway { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NLBBackendSet) DeepCopyInto(out *NLBBackendSet) { + *out = *in + if in.ListenerPort != nil { + in, out := &in.ListenerPort, &out.ListenerPort + *out = new(int32) + **out = **in + } + in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NLBBackendSet. +func (in *NLBBackendSet) DeepCopy() *NLBBackendSet { + if in == nil { + return nil + } + out := new(NLBBackendSet) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NLBSpec) DeepCopyInto(out *NLBSpec) { *out = *in + if in.BackendSets != nil { + in, out := &in.BackendSets, &out.BackendSets + *out = make([]NLBBackendSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.BackendSetDetails.DeepCopyInto(&out.BackendSetDetails) if in.ReservedIpIds != nil { in, out := &in.ReservedIpIds, &out.ReservedIpIds @@ -1864,21 +1892,6 @@ func (in *OCIClusterTemplateSpec) DeepCopy() *OCIClusterTemplateSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OCIClusterWebhook) DeepCopyInto(out *OCIClusterWebhook) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCIClusterWebhook. -func (in *OCIClusterWebhook) DeepCopy() *OCIClusterWebhook { - if in == nil { - return nil - } - out := new(OCIClusterWebhook) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OCIMachine) DeepCopyInto(out *OCIMachine) { *out = *in @@ -2452,21 +2465,6 @@ func (in *OCIManagedClusterTemplateSpec) DeepCopy() *OCIManagedClusterTemplateSp return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OCIManagedClusterWebhook) DeepCopyInto(out *OCIManagedClusterWebhook) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCIManagedClusterWebhook. -func (in *OCIManagedClusterWebhook) DeepCopy() *OCIManagedClusterWebhook { - if in == nil { - return nil - } - out := new(OCIManagedClusterWebhook) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OCIManagedControlPlane) DeepCopyInto(out *OCIManagedControlPlane) { *out = *in diff --git a/cloud/ociutil/ociutil.go b/cloud/ociutil/ociutil.go index e49fb265d..05b85b5c6 100644 --- a/cloud/ociutil/ociutil.go +++ b/cloud/ociutil/ociutil.go @@ -36,13 +36,14 @@ import ( ) const ( - WorkRequestPollInterval = 5 * time.Second - WorkRequestTimeout = 2 * time.Minute - MaxOPCRetryTokenBytes = 64 - CreatedBy = "CreatedBy" - OCIClusterAPIProvider = "OCIClusterAPIProvider" - ClusterResourceIdentifier = "ClusterResourceIdentifier" - OutOfHostCapacityErr = "Out of host capacity" + DefaultAPIServerPort int32 = 6443 + WorkRequestPollInterval = 5 * time.Second + WorkRequestTimeout = 2 * time.Minute + MaxOPCRetryTokenBytes = 64 + CreatedBy = "CreatedBy" + OCIClusterAPIProvider = "OCIClusterAPIProvider" + ClusterResourceIdentifier = "ClusterResourceIdentifier" + OutOfHostCapacityErr = "Out of host capacity" ) // ErrNotFound is for simulation during testing, OCI SDK does not have a way diff --git a/cloud/scope/apiserver_lb_backendsets_test.go b/cloud/scope/apiserver_lb_backendsets_test.go new file mode 100644 index 000000000..24a035308 --- /dev/null +++ b/cloud/scope/apiserver_lb_backendsets_test.go @@ -0,0 +1,169 @@ +package scope + +import ( + "testing" + + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" +) + +func TestBuildDesiredNLBListenersAndBackendSets(t *testing.T) { + scope := &ClusterScope{Cluster: &clusterv1.Cluster{}} + lb := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: "primary-set"}, + { + Name: "rollout-set", + ListenerPort: int32Ptr(9345), + BackendSetDetails: infrastructurev1beta2.BackendSetDetails{ + HealthChecker: infrastructurev1beta2.HealthChecker{ + UrlPath: common.String("/readyz"), + }, + }, + }, + }, + }, + } + + listeners, backendSets := scope.buildDesiredNLBListenersAndBackendSets(lb) + if len(backendSets) != 2 { + t.Fatalf("expected two backend sets, got %#v", backendSets) + } + if len(listeners) != 2 { + t.Fatalf("expected two listeners, got %#v", listeners) + } + if listeners[APIServerLBListener].DefaultBackendSetName == nil || *listeners[APIServerLBListener].DefaultBackendSetName != "primary-set" { + t.Fatalf("expected listener to reference first backend set, got %#v", listeners[APIServerLBListener]) + } + secondaryListenerName := desiredAPIServerListenerName(1, 2, "rollout-set") + if listeners[secondaryListenerName].Port == nil || *listeners[secondaryListenerName].Port != 9345 { + t.Fatalf("expected secondary listener to use custom port, got %#v", listeners[secondaryListenerName]) + } + if backendSets["rollout-set"].HealthChecker == nil || backendSets["rollout-set"].HealthChecker.UrlPath == nil || *backendSets["rollout-set"].HealthChecker.UrlPath != "/readyz" { + t.Fatalf("expected rollout backend set health checker override, got %#v", backendSets["rollout-set"]) + } +} + +func int32Ptr(v int32) *int32 { + return &v +} + +func TestBuildDesiredLBListenersAndBackendSets(t *testing.T) { + scope := &ClusterScope{Cluster: &clusterv1.Cluster{}} + lb := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: "primary-set"}, + {Name: "rollout-set"}, + }, + }, + } + + listeners, backendSets := scope.buildDesiredLBListenersAndBackendSets(lb) + if len(backendSets) != 2 { + t.Fatalf("expected two backend sets, got %#v", backendSets) + } + if len(listeners) != 2 { + t.Fatalf("expected two listeners, got %#v", listeners) + } + if listeners[APIServerLBListener].DefaultBackendSetName == nil || *listeners[APIServerLBListener].DefaultBackendSetName != "primary-set" { + t.Fatalf("expected listener to reference first backend set, got %#v", listeners[APIServerLBListener]) + } +} + +func TestLBSpecPreservesBackendSets(t *testing.T) { + secondaryPort := int32(9345) + scope := &ClusterScope{ + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{ + OCICluster: &infrastructurev1beta2.OCICluster{ + Spec: infrastructurev1beta2.OCIClusterSpec{ + NetworkSpec: infrastructurev1beta2.NetworkSpec{ + APIServerLB: infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "apiserver-lb-backendset-2", ListenerPort: &secondaryPort}, + }, + }, + }, + }, + }, + }, + }, + } + + got := scope.DesiredAPIServerLoadBalancer() + if len(got.NLBSpec.BackendSets) != 2 { + t.Fatalf("expected backend sets to be preserved, got %#v", got.NLBSpec.BackendSets) + } + if got.NLBSpec.BackendSets[1].Name != "apiserver-lb-backendset-2" { + t.Fatalf("expected secondary backend set to be preserved, got %#v", got.NLBSpec.BackendSets) + } +} + +func TestIsNLBEqual_DetectsSpecOnlyResourceChanges(t *testing.T) { + secondaryPort := int32(9345) + scope := &ClusterScope{Cluster: &clusterv1.Cluster{}} + desired := infrastructurev1beta2.LoadBalancer{ + Name: "cluster-apiserver", + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "rollout-set", ListenerPort: &secondaryPort}, + }, + }, + } + + actual := &networkloadbalancer.NetworkLoadBalancer{ + DisplayName: common.String("cluster-apiserver"), + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + desiredAPIServerListenerName(1, 2, "rollout-set"): { + Name: common.String(desiredAPIServerListenerName(1, 2, "rollout-set")), + DefaultBackendSetName: common.String("rollout-set"), + Port: common.Int(9345), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + "rollout-set": { + Name: common.String("rollout-set"), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + }, + } + if !scope.IsNLBEqual(actual, desired) { + t.Fatalf("expected matching NLB resources to be equal") + } + + delete(actual.Listeners, desiredAPIServerListenerName(1, 2, "rollout-set")) + delete(actual.BackendSets, "rollout-set") + if scope.IsNLBEqual(actual, desired) { + t.Fatalf("expected spec-only listener/backend-set changes to make NLB unequal") + } +} diff --git a/cloud/scope/apiserver_listener_naming.go b/cloud/scope/apiserver_listener_naming.go new file mode 100644 index 000000000..b075cc953 --- /dev/null +++ b/cloud/scope/apiserver_listener_naming.go @@ -0,0 +1,33 @@ +package scope + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" +) + +func desiredAPIServerListenerPort(defaultPort int32, backendSet infrastructurev1beta2.NLBBackendSet) int32 { + if backendSet.ListenerPort != nil { + return *backendSet.ListenerPort + } + return defaultPort +} + +func desiredAPIServerListenerName(index int, total int, backendSetName string) string { + // Preserve legacy name for legacy/single-backend-set behavior. + if total == 1 && backendSetName == APIServerLBBackendSetName { + return APIServerLBListener + } + // Keep the first listener stable for the control plane endpoint. + if index == 0 { + return APIServerLBListener + } + return fmt.Sprintf("%s-%s", APIServerLBListener, stableShortHash(backendSetName)) +} + +func stableShortHash(value string) string { + sum := sha1.Sum([]byte(value)) + return hex.EncodeToString(sum[:])[:8] +} diff --git a/cloud/scope/apiserver_listener_naming_test.go b/cloud/scope/apiserver_listener_naming_test.go new file mode 100644 index 000000000..c3f0c9a6d --- /dev/null +++ b/cloud/scope/apiserver_listener_naming_test.go @@ -0,0 +1,39 @@ +package scope + +import ( + "testing" + + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" +) + +func TestDesiredAPIServerListenerName(t *testing.T) { + t.Run("keeps legacy listener name for single legacy backend set", func(t *testing.T) { + got := desiredAPIServerListenerName(0, 1, APIServerLBBackendSetName) + if got != APIServerLBListener { + t.Fatalf("expected legacy listener name %q, got %q", APIServerLBListener, got) + } + }) + + t.Run("uses stable hashed names for secondary listeners", func(t *testing.T) { + a := desiredAPIServerListenerName(1, 2, "rollout-set") + b := desiredAPIServerListenerName(1, 2, "rollout-set") + c := desiredAPIServerListenerName(1, 2, "another-set") + if a != b { + t.Fatalf("expected deterministic name for same backend set, got %q and %q", a, b) + } + if a == c { + t.Fatalf("expected different names for different backend sets, got %q and %q", a, c) + } + }) +} + +func TestDesiredAPIServerListenerPort(t *testing.T) { + defaultPort := int32(6443) + if got := desiredAPIServerListenerPort(defaultPort, infrastructurev1beta2.NLBBackendSet{}); got != defaultPort { + t.Fatalf("expected default port %d, got %d", defaultPort, got) + } + customPort := int32(9345) + if got := desiredAPIServerListenerPort(defaultPort, infrastructurev1beta2.NLBBackendSet{ListenerPort: &customPort}); got != customPort { + t.Fatalf("expected custom port %d, got %d", customPort, got) + } +} diff --git a/cloud/scope/defaults.go b/cloud/scope/defaults.go index 77c97e3b9..c062aae6d 100644 --- a/cloud/scope/defaults.go +++ b/cloud/scope/defaults.go @@ -16,7 +16,10 @@ package scope -import "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" +import ( + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" +) const ( VcnDefaultCidr = "10.0.0.0/16" @@ -24,7 +27,7 @@ const ( ControlPlaneMachineSubnetDefaultCIDR = "10.0.0.0/29" WorkerSubnetDefaultCIDR = "10.0.64.0/20" ServiceLoadBalancerDefaultCIDR = "10.0.0.32/27" - ApiServerPort = 6443 + ApiServerPort = ociutil.DefaultAPIServerPort APIServerLBBackendSetName = "apiserver-lb-backendset" APIServerLBListener = "apiserver-lb-listener" SGWServiceSuffix = "-services-in-oracle-services-network" diff --git a/cloud/scope/lb_resource_reconcile_errors.go b/cloud/scope/lb_resource_reconcile_errors.go new file mode 100644 index 000000000..13f9bda59 --- /dev/null +++ b/cloud/scope/lb_resource_reconcile_errors.go @@ -0,0 +1,22 @@ +package scope + +import ( + "net/http" + "strings" + + "github.com/oracle/oci-go-sdk/v65/common" +) + +func isAlreadyExistsOCIError(err error) bool { + serviceErr, ok := common.IsServiceError(err) + if !ok { + return false + } + if serviceErr.GetHTTPStatusCode() == http.StatusConflict { + return true + } + + code := strings.ToLower(serviceErr.GetCode()) + message := strings.ToLower(serviceErr.GetMessage()) + return strings.Contains(code, "already") || strings.Contains(message, "already exists") +} diff --git a/cloud/scope/load_balancer_reconciler.go b/cloud/scope/load_balancer_reconciler.go index e7b51fe3b..76a6490fb 100644 --- a/cloud/scope/load_balancer_reconciler.go +++ b/cloud/scope/load_balancer_reconciler.go @@ -31,7 +31,7 @@ import ( // ReconcileApiServerLB tries to move the Load Balancer to the desired OCICluster Spec func (s *ClusterScope) ReconcileApiServerLB(ctx context.Context) error { - desiredApiServerLb := s.LBSpec() + desiredApiServerLb := s.DesiredAPIServerLoadBalancer() lb, err := s.GetLoadBalancers(ctx) if err != nil { @@ -68,7 +68,10 @@ func (s *ClusterScope) ReconcileApiServerLB(ctx context.Context) error { Host: *lbIP, Port: s.APIServerPort(), }) - return err + if err := s.reconcileLBResources(ctx, desiredApiServerLb); err != nil { + return err + } + return nil } // DeleteApiServerLB retrieves and attempts to delete the Load Balancer if found. @@ -97,12 +100,12 @@ func (s *ClusterScope) DeleteApiServerLB(ctx context.Context) error { return nil } -// LBSpec builds the LoadBalancer from the ClusterScope and returns it -func (s *ClusterScope) LBSpec() infrastructurev1beta2.LoadBalancer { - lbSpec := infrastructurev1beta2.LoadBalancer{ - Name: s.GetControlPlaneLoadBalancerName(), - } - return lbSpec +// DesiredAPIServerLoadBalancer builds the desired LBaaS load balancer from the ClusterScope. +func (s *ClusterScope) DesiredAPIServerLoadBalancer() infrastructurev1beta2.LoadBalancer { + return infrastructurev1beta2.NewAPIServerLoadBalancer( + s.GetControlPlaneLoadBalancerName(), + *s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.APIServerLoadBalancerSpec(), + ) } // GetControlPlaneLoadBalancerName returns the user defined APIServerLB name from the spec or @@ -135,6 +138,165 @@ func (s *ClusterScope) UpdateLB(ctx context.Context, lb infrastructurev1beta2.Lo s.Logger.Error(err, "failed to reconcile the apiserver LB, failed to update lb") return errors.Wrap(err, "failed to reconcile the apiserver LB, failed to update lb") } + if err := s.reconcileLBResources(ctx, lb); err != nil { + return err + } + return nil +} + +func (s *ClusterScope) reconcileLBResources(ctx context.Context, lb infrastructurev1beta2.LoadBalancer) error { + lbID := s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.LoadBalancerId + actualResp, err := s.LoadBalancerClient.GetLoadBalancer(ctx, loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: lbID, + }) + if err != nil { + return errors.Wrap(err, "failed to get lb for resource reconciliation") + } + desiredListeners, desiredBackendSets := s.buildDesiredLBListenersAndBackendSets(lb) + + if err := reconcileNamedResources( + actualResp.LoadBalancer.BackendSets, + desiredBackendSets, + func(name string, desired loadbalancer.BackendSetDetails) error { + resp, err := s.LoadBalancerClient.CreateBackendSet(ctx, loadbalancer.CreateBackendSetRequest{ + LoadBalancerId: lbID, + CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ + Name: common.String(name), + Policy: desired.Policy, + HealthChecker: desired.HealthChecker, + }, + }) + if err != nil { + if isAlreadyExistsOCIError(err) { + s.Logger.Info("LB backend set already exists during reconcile; continuing", "backendSet", name) + return nil + } + return errors.Wrapf(err, "failed to create lb backend set %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting create backend set %q", name) + } + return nil + }, + lbBackendSetNeedsUpdate, + func(name string, desired loadbalancer.BackendSetDetails) error { + resp, err := s.LoadBalancerClient.UpdateBackendSet(ctx, loadbalancer.UpdateBackendSetRequest{ + LoadBalancerId: lbID, + BackendSetName: common.String(name), + UpdateBackendSetDetails: loadbalancer.UpdateBackendSetDetails{ + Policy: desired.Policy, + Backends: []loadbalancer.BackendDetails{}, + HealthChecker: desired.HealthChecker, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update lb backend set %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update backend set %q", name) + } + return nil + }, + ); err != nil { + return err + } + + if err := reconcileNamedResources( + actualResp.LoadBalancer.Listeners, + desiredListeners, + func(name string, desired loadbalancer.ListenerDetails) error { + resp, err := s.LoadBalancerClient.CreateListener(ctx, loadbalancer.CreateListenerRequest{ + LoadBalancerId: lbID, + CreateListenerDetails: loadbalancer.CreateListenerDetails{ + Name: common.String(name), + DefaultBackendSetName: desired.DefaultBackendSetName, + Port: desired.Port, + Protocol: desired.Protocol, + }, + }) + if err != nil { + if isAlreadyExistsOCIError(err) { + s.Logger.Info("LB listener already exists during reconcile; continuing", "listener", name) + return nil + } + return errors.Wrapf(err, "failed to create lb listener %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting create listener %q", name) + } + return nil + }, + lbListenerNeedsUpdate, + func(name string, desired loadbalancer.ListenerDetails) error { + resp, err := s.LoadBalancerClient.UpdateListener(ctx, loadbalancer.UpdateListenerRequest{ + LoadBalancerId: lbID, + ListenerName: common.String(name), + UpdateListenerDetails: loadbalancer.UpdateListenerDetails{ + DefaultBackendSetName: desired.DefaultBackendSetName, + Port: desired.Port, + Protocol: desired.Protocol, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update lb listener %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update listener %q", name) + } + return nil + }, + ); err != nil { + return err + } + staleListenerDeleted, err := deleteStaleNamedResources( + actualResp.LoadBalancer.Listeners, + desiredNameSet(desiredListeners), + func(loadbalancer.Listener) bool { return true }, + func(name string, _ loadbalancer.Listener) error { + resp, err := s.LoadBalancerClient.DeleteListener(ctx, loadbalancer.DeleteListenerRequest{ + LoadBalancerId: lbID, + ListenerName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale lb listener %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete listener %q", name) + } + return nil + }, + ) + if err != nil { + return err + } + if staleListenerDeleted { + actualResp, err = s.LoadBalancerClient.GetLoadBalancer(ctx, loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: lbID, + }) + if err != nil { + return errors.Wrap(err, "failed to refresh lb after stale listener reconciliation") + } + } + if _, err := deleteStaleNamedResources( + actualResp.LoadBalancer.BackendSets, + desiredNameSet(desiredBackendSets), + func(actual loadbalancer.BackendSet) bool { return len(actual.Backends) == 0 }, + func(name string, _ loadbalancer.BackendSet) error { + resp, err := s.LoadBalancerClient.DeleteBackendSet(ctx, loadbalancer.DeleteBackendSetRequest{ + LoadBalancerId: lbID, + BackendSetName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale lb backend set %q", name) + } + if _, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete backend set %q", name) + } + return nil + }, + ); err != nil { + return err + } return nil } @@ -145,21 +307,7 @@ func (s *ClusterScope) UpdateLB(ctx context.Context, lb infrastructurev1beta2.Lo // See https://docs.oracle.com/en-us/iaas/Content/LoadBalancer/overview.htm for more details on the Network // Load Balancer func (s *ClusterScope) CreateLB(ctx context.Context, lb infrastructurev1beta2.LoadBalancer) (*string, *string, error) { - listenerDetails := make(map[string]loadbalancer.ListenerDetails) - listenerDetails[APIServerLBListener] = loadbalancer.ListenerDetails{ - Protocol: common.String("TCP"), - Port: common.Int(int(s.APIServerPort())), - DefaultBackendSetName: common.String(APIServerLBBackendSetName), - } - backendSetDetails := make(map[string]loadbalancer.BackendSetDetails) - backendSetDetails[APIServerLBBackendSetName] = loadbalancer.BackendSetDetails{ - Policy: common.String("ROUND_ROBIN"), - HealthChecker: &loadbalancer.HealthCheckerDetails{ - Port: common.Int(int(s.APIServerPort())), - Protocol: common.String("TCP"), - }, - Backends: []loadbalancer.BackendDetails{}, - } + listenerDetails, backendSetDetails := s.buildDesiredLBListenersAndBackendSets(lb) var controlPlaneEndpointSubnets []string for _, subnet := range ptr.ToSubnetSlice(s.OCIClusterAccessor.GetNetworkSpec().Vcn.Subnets) { if subnet.ID != nil && subnet.Role == infrastructurev1beta2.ControlPlaneEndpointRole { @@ -225,6 +373,34 @@ func (s *ClusterScope) CreateLB(ctx context.Context, lb infrastructurev1beta2.Lo return lbs.Id, lbIp, nil } +func (s *ClusterScope) buildDesiredLBListenersAndBackendSets(lb infrastructurev1beta2.LoadBalancer) (map[string]loadbalancer.ListenerDetails, map[string]loadbalancer.BackendSetDetails) { + canonicalBackendSets := desiredAPIServerBackendSets(lb) + + backendSetDetails := make(map[string]loadbalancer.BackendSetDetails, len(canonicalBackendSets)) + for _, backendSet := range canonicalBackendSets { + backendSetDetails[backendSet.Name] = loadbalancer.BackendSetDetails{ + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthCheckerDetails{ + Port: common.Int(int(s.APIServerPort())), + Protocol: common.String("TCP"), + }, + Backends: []loadbalancer.BackendDetails{}, + } + } + + listenerDetails := make(map[string]loadbalancer.ListenerDetails, len(canonicalBackendSets)) + for i, backendSet := range canonicalBackendSets { + listenerName := desiredAPIServerListenerName(i, len(canonicalBackendSets), backendSet.Name) + port := desiredAPIServerListenerPort(s.APIServerPort(), backendSet) + listenerDetails[listenerName] = loadbalancer.ListenerDetails{ + Protocol: common.String("TCP"), + Port: common.Int(int(port)), + DefaultBackendSetName: common.String(backendSet.Name), + } + } + return listenerDetails, backendSetDetails +} + func (s *ClusterScope) getLoadbalancerIp(lb loadbalancer.LoadBalancer) (*string, error) { var lbIp *string if len(lb.IpAddresses) < 1 { @@ -246,14 +422,55 @@ func (s *ClusterScope) getLoadbalancerIp(lb loadbalancer.LoadBalancer) (*string, } // IsLBEqual determines if the actual loadbalancer.LoadBalancer is equal to the desired. -// Equality is determined by DisplayName, FreeformTags and DefinedTags matching. +// Equality is determined by the LB identity plus desired listener/backend-set resources matching. func (s *ClusterScope) IsLBEqual(actual *loadbalancer.LoadBalancer, desired infrastructurev1beta2.LoadBalancer) bool { if desired.Name != *actual.DisplayName { return false } + + desiredListeners, desiredBackendSets := s.buildDesiredLBListenersAndBackendSets(desired) + if len(actual.Listeners) != len(desiredListeners) { + return false + } + for name, desiredListener := range desiredListeners { + actualListener, exists := actual.Listeners[name] + if !exists { + return false + } + if lbListenerNeedsUpdate(actualListener, desiredListener) { + return false + } + } + + if len(actual.BackendSets) != len(desiredBackendSets) { + return false + } + for name, desiredBackendSet := range desiredBackendSets { + actualBackendSet, exists := actual.BackendSets[name] + if !exists { + return false + } + if lbBackendSetNeedsUpdate(actualBackendSet, desiredBackendSet) { + return false + } + } + return true } +func lbListenerNeedsUpdate(actual loadbalancer.Listener, desired loadbalancer.ListenerDetails) bool { + return !intPtrEqual(actual.Port, desired.Port) || + !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || + !ptr.StringEquals(actual.Protocol, ptr.ToString(desired.Protocol)) +} + +func lbBackendSetNeedsUpdate(actual loadbalancer.BackendSet, desired loadbalancer.BackendSetDetails) bool { + return !ptr.StringEquals(actual.Policy, ptr.ToString(desired.Policy)) || + actual.HealthChecker == nil || desired.HealthChecker == nil || + !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || + !ptr.StringEquals(actual.HealthChecker.Protocol, ptr.ToString(desired.HealthChecker.Protocol)) +} + // GetLoadBalancers retrieves the Cluster's loadbalancer.LoadBalancer using the one of the following methods // // 1. the OCICluster's spec LoadBalancerId diff --git a/cloud/scope/load_balancer_reconciler_test.go b/cloud/scope/load_balancer_reconciler_test.go index 8ac36faa2..dd71eb2bd 100644 --- a/cloud/scope/load_balancer_reconciler_test.go +++ b/cloud/scope/load_balancer_reconciler_test.go @@ -103,6 +103,24 @@ func TestLBReconciliation(t *testing.T) { DefinedTags: make(map[string]map[string]interface{}), IsPrivate: common.Bool(false), DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + }, IpAddresses: []loadbalancer.IpAddress{ { IpAddress: common.String("2.2.2.2"), @@ -316,6 +334,36 @@ func TestLBReconciliation(t *testing.T) { }, }, }, nil) + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })). + Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Id: common.String("lb-id"), + FreeformTags: tags, + IsPrivate: common.Bool(false), + DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + }, + IpAddresses: []loadbalancer.IpAddress{{IpAddress: common.String("2.2.2.2"), IsPublic: common.Bool(true)}}, + }, + }, nil) }, }, @@ -411,6 +459,24 @@ func TestLBReconciliation(t *testing.T) { }, }, }, nil) + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })). + Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Id: common.String("lb-id"), + FreeformTags: tags, + IsPrivate: common.Bool(false), + DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: {Name: common.String(APIServerLBListener), Protocol: common.String("TCP"), Port: common.Int(6443), DefaultBackendSetName: common.String(APIServerLBBackendSetName)}, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: {Name: common.String(APIServerLBBackendSetName), Policy: common.String("ROUND_ROBIN"), HealthChecker: &loadbalancer.HealthChecker{Port: common.Int(6443), Protocol: common.String("TCP")}}, + }, + IpAddresses: []loadbalancer.IpAddress{{IpAddress: common.String("2.2.2.2"), IsPublic: common.Bool(true)}}, + }, + }, nil) }, }, @@ -932,3 +998,297 @@ func TestLBDeletion(t *testing.T) { }) } } + +func TestReconcileLBResources_CreatesBackendSetsBeforeListeners(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + lbClient := mock_lb.NewMockLoadBalancerClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("lb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + LoadBalancerClient: lbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{OCICluster: ociCluster}, + Client: client, + }) + g.Expect(err).To(BeNil()) + + secondaryPort := int32(9345) + desiredLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "apiserver-lb-backendset-2", ListenerPort: &secondaryPort}, + }, + }, + } + secondaryListenerName := desiredAPIServerListenerName(1, 2, "apiserver-lb-backendset-2") + + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })).Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + }, + }, + }, nil) + + gomock.InOrder( + lbClient.EXPECT().CreateBackendSet(gomock.Any(), gomock.Eq(loadbalancer.CreateBackendSetRequest{ + LoadBalancerId: common.String("lb-id"), + CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ + Name: common.String("apiserver-lb-backendset-2"), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthCheckerDetails{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + })).Return(loadbalancer.CreateBackendSetResponse{OpcWorkRequestId: common.String("wr-backend-set")}, nil), + lbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(loadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-backend-set"), + })).Return(loadbalancer.GetWorkRequestResponse{ + WorkRequest: loadbalancer.WorkRequest{LifecycleState: loadbalancer.WorkRequestLifecycleStateSucceeded}, + }, nil), + lbClient.EXPECT().CreateListener(gomock.Any(), gomock.Eq(loadbalancer.CreateListenerRequest{ + LoadBalancerId: common.String("lb-id"), + CreateListenerDetails: loadbalancer.CreateListenerDetails{ + Name: common.String(secondaryListenerName), + DefaultBackendSetName: common.String("apiserver-lb-backendset-2"), + Port: common.Int(9345), + Protocol: common.String("TCP"), + }, + })).Return(loadbalancer.CreateListenerResponse{OpcWorkRequestId: common.String("wr-listener")}, nil), + lbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(loadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-listener"), + })).Return(loadbalancer.GetWorkRequestResponse{ + WorkRequest: loadbalancer.WorkRequest{LifecycleState: loadbalancer.WorkRequestLifecycleStateSucceeded}, + }, nil), + ) + + g.Expect(cs.reconcileLBResources(context.Background(), desiredLB)).To(Succeed()) +} + +func TestReconcileLBResources_IgnoresAlreadyExistsOnCreate(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + lbClient := mock_lb.NewMockLoadBalancerClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("lb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + LoadBalancerClient: lbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{OCICluster: ociCluster}, + Client: client, + }) + g.Expect(err).To(BeNil()) + + secondaryPort := int32(9345) + desiredLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "apiserver-lb-backendset-2", ListenerPort: &secondaryPort}, + }, + }, + } + secondaryListenerName := desiredAPIServerListenerName(1, 2, "apiserver-lb-backendset-2") + + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })).Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + }, + }, + }, nil) + + lbClient.EXPECT().CreateBackendSet(gomock.Any(), gomock.Eq(loadbalancer.CreateBackendSetRequest{ + LoadBalancerId: common.String("lb-id"), + CreateBackendSetDetails: loadbalancer.CreateBackendSetDetails{ + Name: common.String("apiserver-lb-backendset-2"), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthCheckerDetails{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + })).Return(loadbalancer.CreateBackendSetResponse{}, mockServiceError{ + status: 409, + code: "Conflict", + message: "backend set already exists", + }) + lbClient.EXPECT().CreateListener(gomock.Any(), gomock.Eq(loadbalancer.CreateListenerRequest{ + LoadBalancerId: common.String("lb-id"), + CreateListenerDetails: loadbalancer.CreateListenerDetails{ + Name: common.String(secondaryListenerName), + DefaultBackendSetName: common.String("apiserver-lb-backendset-2"), + Port: common.Int(9345), + Protocol: common.String("TCP"), + }, + })).Return(loadbalancer.CreateListenerResponse{}, mockServiceError{ + status: 409, + code: "Conflict", + message: "listener already exists", + }) + + g.Expect(cs.reconcileLBResources(context.Background(), desiredLB)).To(Succeed()) +} + +func TestReconcileLBResources_RefreshesBeforeDeletingStaleBackendSets(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + lbClient := mock_lb.NewMockLoadBalancerClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("lb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + LoadBalancerClient: lbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{OCICluster: ociCluster}, + Client: client, + }) + g.Expect(err).To(BeNil()) + + desiredLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + }, + }, + } + + gomock.InOrder( + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })).Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + "stale-listener": {Name: common.String("stale-listener")}, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + "stale-set": {Name: common.String("stale-set"), Backends: []loadbalancer.Backend{}}, + }, + }, + }, nil), + lbClient.EXPECT().DeleteListener(gomock.Any(), gomock.Eq(loadbalancer.DeleteListenerRequest{ + LoadBalancerId: common.String("lb-id"), + ListenerName: common.String("stale-listener"), + })).Return(loadbalancer.DeleteListenerResponse{OpcWorkRequestId: common.String("wr-delete-listener")}, nil), + lbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(loadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-delete-listener"), + })).Return(loadbalancer.GetWorkRequestResponse{ + WorkRequest: loadbalancer.WorkRequest{LifecycleState: loadbalancer.WorkRequestLifecycleStateSucceeded}, + }, nil), + lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String("lb-id"), + })).Return(loadbalancer.GetLoadBalancerResponse{ + LoadBalancer: loadbalancer.LoadBalancer{ + Listeners: map[string]loadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + Protocol: common.String("TCP"), + Port: common.Int(6443), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + }, + }, + BackendSets: map[string]loadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: common.String("ROUND_ROBIN"), + HealthChecker: &loadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: common.String("TCP"), + }, + }, + "stale-set": { + Name: common.String("stale-set"), + Backends: []loadbalancer.Backend{ + {Name: common.String("10.0.0.4:6443")}, + }, + }, + }, + }, + }, nil), + ) + + g.Expect(cs.reconcileLBResources(context.Background(), desiredLB)).To(Succeed()) +} diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index 2048ad840..492880a60 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -634,35 +634,43 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { if err != nil { return err } - backendSet := lb.BackendSets[APIServerLBBackendSetName] - // When creating a LB, there is no way to set the backend Name, default backend name is the instance IP and port - // So we use default backend name instead of machine name - backendName := instanceIp + ":" + strconv.Itoa(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)) - if !m.containsLBBackend(backendSet, backendName) { - logger := m.Logger.WithValues("backend-set", *backendSet.Name) - logger.Info("Checking work request status for create backend") - // we always try to create the backend if it exists during a reconcile loop and wait for the work request - // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the - // work request, the create backend call may throw an error which is ok, as reconcile will go into - // an exponential backoff - resp, err := m.LoadBalancerClient.CreateBackend(ctx, loadbalancer.CreateBackendRequest{ - LoadBalancerId: loadbalancerId, - BackendSetName: backendSet.Name, - CreateBackendDetails: loadbalancer.CreateBackendDetails{ - IpAddress: common.String(instanceIp), - Port: common.Int(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)), - }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", string(m.OCIMachine.UID)), - }) - if err != nil { - return err + for _, backendSetName := range m.desiredAPIServerBackendSetNames() { + backendSet, ok := lb.BackendSets[backendSetName] + if !ok { + return errors.Errorf("backend set %q not found on load balancer", backendSetName) } - m.OCIMachine.Status.CreateBackendWorkRequestId = *resp.OpcWorkRequestId - logger.Info("Add instance to LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) - logger.Info("Waiting for LB work request to be complete") - _, err = ociutil.AwaitLBWorkRequest(ctx, m.LoadBalancerClient, resp.OpcWorkRequestId) - if err != nil { - return err + if backendSet.Name == nil { + backendSet.Name = common.String(backendSetName) + } + backendPort := m.desiredAPIServerBackendPort(backendSetName) + // For LBaaS backend names are represented as ":". + backendName := instanceIp + ":" + strconv.Itoa(int(backendPort)) + if !m.containsLBBackend(backendSet, backendName) { + logger := m.Logger.WithValues("backend-set", *backendSet.Name) + logger.Info("Checking work request status for create backend") + // we always try to create the backend if it exists during a reconcile loop and wait for the work request + // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the + // work request, the create backend call may throw an error which is ok, as reconcile will go into + // an exponential backoff + resp, err := m.LoadBalancerClient.CreateBackend(ctx, loadbalancer.CreateBackendRequest{ + LoadBalancerId: loadbalancerId, + BackendSetName: backendSet.Name, + CreateBackendDetails: loadbalancer.CreateBackendDetails{ + IpAddress: common.String(instanceIp), + Port: common.Int(int(backendPort)), + }, + OpcRetryToken: m.createBackendRetryToken(backendSetName, backendPort), + }) + if err != nil { + return err + } + m.OCIMachine.Status.CreateBackendWorkRequestId = *resp.OpcWorkRequestId + logger.Info("Add instance to LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) + logger.Info("Waiting for LB work request to be complete") + _, err = ociutil.AwaitLBWorkRequest(ctx, m.LoadBalancerClient, resp.OpcWorkRequestId) + if err != nil { + return err + } } } @@ -673,35 +681,47 @@ func (m *MachineScope) ReconcileCreateInstanceOnLB(ctx context.Context) error { if err != nil { return err } - backendSet := lb.BackendSets[APIServerLBBackendSetName] - if !m.containsNLBBackend(backendSet, m.Name()) { - logger := m.Logger.WithValues("backend-set", *backendSet.Name) - logger.Info("Checking work request status for create backend") - // we always try to create the backend if it exists during a reconcile loop and wait for the work request - // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the - // work request, the create backend call may throw an error which is ok, as reconcile will go into - // an exponential backoff - resp, err := m.NetworkLoadBalancerClient.CreateBackend(ctx, networkloadbalancer.CreateBackendRequest{ - NetworkLoadBalancerId: loadbalancerId, - BackendSetName: backendSet.Name, - CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ - IpAddress: common.String(instanceIp), - Port: common.Int(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)), - Name: common.String(m.Name()), - }, - OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-backend", string(m.OCIMachine.UID)), - }) - if err != nil { - return err + desiredBackendSetNames := m.desiredAPIServerBackendSetNames() + for _, backendSetName := range desiredBackendSetNames { + backendSet, ok := lb.BackendSets[backendSetName] + if !ok { + return errors.Errorf("backend set %q not found on network load balancer", backendSetName) } - m.OCIMachine.Status.CreateBackendWorkRequestId = *resp.OpcWorkRequestId - logger.Info("Add instance to NLB backend-set", "WorkRequestId", resp.OpcWorkRequestId) - logger.Info("Waiting for NLB work request to be complete") - _, err = ociutil.AwaitNLBWorkRequest(ctx, m.NetworkLoadBalancerClient, resp.OpcWorkRequestId) - if err != nil { - return err + if backendSet.Name == nil { + backendSet.Name = common.String(backendSetName) + } + desiredBackendName := m.desiredNLBBackendName(backendSetName, desiredBackendSetNames) + legacyBackendName := m.Name() + backendPort := m.desiredAPIServerBackendPort(backendSetName) + if !m.containsNLBBackend(backendSet, desiredBackendName) && !m.containsNLBBackend(backendSet, legacyBackendName) { + logger := m.Logger.WithValues("backend-set", *backendSet.Name) + logger.Info("Checking work request status for create backend") + // we always try to create the backend if it exists during a reconcile loop and wait for the work request + // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the + // work request, the create backend call may throw an error which is ok, as reconcile will go into + // an exponential backoff + resp, err := m.NetworkLoadBalancerClient.CreateBackend(ctx, networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: loadbalancerId, + BackendSetName: backendSet.Name, + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String(instanceIp), + Port: common.Int(int(backendPort)), + Name: common.String(desiredBackendName), + }, + OpcRetryToken: m.createBackendRetryToken(backendSetName, backendPort), + }) + if err != nil { + return err + } + m.OCIMachine.Status.CreateBackendWorkRequestId = *resp.OpcWorkRequestId + logger.Info("Add instance to NLB backend-set", "WorkRequestId", resp.OpcWorkRequestId) + logger.Info("Waiting for NLB work request to be complete") + _, err = ociutil.AwaitNLBWorkRequest(ctx, m.NetworkLoadBalancerClient, resp.OpcWorkRequestId) + if err != nil { + return err + } + logger.Info("NLB Backend addition work request is complete") } - logger.Info("NLB Backend addition work request is complete") } } return nil @@ -729,7 +749,6 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { } return err } - backendSet := lb.BackendSets[APIServerLBBackendSetName] // in case of delete from LB backend, if the instance does not have an IP, we consider // the instance to not have been added in first place and hence return nil if len(m.OCIMachine.Status.Addresses) <= 0 { @@ -740,36 +759,48 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { if err != nil { return err } - backendName := instanceIp + ":" + strconv.Itoa(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)) - if m.containsLBBackend(backendSet, backendName) { - logger := m.Logger.WithValues("backend-set", *backendSet.Name) - // in OCI CLI, the colon in the backend name is replaced by %3A - // replace the colon in the backend name by %3A to avoid the error in PCA - escapedBackendName := url.QueryEscape(backendName) - // we always try to delete the backend if it exists during a reconcile loop and wait for the work request - // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the - // work request, the delete backend call may throw an error which is ok, as reconcile will go into - // an exponential backoff - resp, err := m.LoadBalancerClient.DeleteBackend(ctx, loadbalancer.DeleteBackendRequest{ - LoadBalancerId: loadbalancerId, - BackendSetName: backendSet.Name, - BackendName: common.String(escapedBackendName), - }) - if err != nil { - logger.Error(err, "Delete instance from LB backend-set failed", - "backendSetName", *backendSet.Name, - "backendName", escapedBackendName, - ) - return err + for backendSetName, backendSet := range lb.BackendSets { + if backendSet.Name == nil { + backendSet.Name = common.String(backendSetName) } - m.OCIMachine.Status.DeleteBackendWorkRequestId = *resp.OpcWorkRequestId - logger.Info("Delete instance from LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) - logger.Info("Waiting for LB work request to be complete") - _, err = ociutil.AwaitLBWorkRequest(ctx, m.LoadBalancerClient, resp.OpcWorkRequestId) - if err != nil { - return err + desiredBackendName := m.desiredLBBackendName(instanceIp, backendSetName) + legacyBackendName := instanceIp + ":" + strconv.Itoa(int(m.OCIClusterAccessor.GetControlPlaneEndpoint().Port)) + backendNameToDelete := "" + if m.containsLBBackend(backendSet, desiredBackendName) { + backendNameToDelete = desiredBackendName + } else if desiredBackendName != legacyBackendName && m.containsLBBackend(backendSet, legacyBackendName) { + backendNameToDelete = legacyBackendName + } + if backendNameToDelete != "" { + logger := m.Logger.WithValues("backend-set", *backendSet.Name) + // in OCI CLI, the colon in the backend name is replaced by %3A + // replace the colon in the backend name by %3A to avoid the error in PCA + escapedBackendName := url.QueryEscape(backendNameToDelete) + // we always try to delete the backend if it exists during a reconcile loop and wait for the work request + // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the + // work request, the delete backend call may throw an error which is ok, as reconcile will go into + // an exponential backoff + resp, err := m.LoadBalancerClient.DeleteBackend(ctx, loadbalancer.DeleteBackendRequest{ + LoadBalancerId: loadbalancerId, + BackendSetName: backendSet.Name, + BackendName: common.String(escapedBackendName), + }) + if err != nil { + logger.Error(err, "Delete instance from LB backend-set failed", + "backendSetName", *backendSet.Name, + "backendName", escapedBackendName, + ) + return err + } + m.OCIMachine.Status.DeleteBackendWorkRequestId = *resp.OpcWorkRequestId + logger.Info("Delete instance from LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) + logger.Info("Waiting for LB work request to be complete") + _, err = ociutil.AwaitLBWorkRequest(ctx, m.LoadBalancerClient, resp.OpcWorkRequestId) + if err != nil { + return err + } + logger.Info("LB Backend addition work request is complete") } - logger.Info("LB Backend addition work request is complete") } } else { lb, err := m.NetworkLoadBalancerClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ @@ -782,33 +813,92 @@ func (m *MachineScope) ReconcileDeleteInstanceOnLB(ctx context.Context) error { } return err } - backendSet := lb.BackendSets[APIServerLBBackendSetName] - if m.containsNLBBackend(backendSet, m.Name()) { - logger := m.Logger.WithValues("backend-set", *backendSet.Name) - // we always try to delete the backend if it exists during a reconcile loop and wait for the work request - // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the - // work request, the delete backend call may throw an error which is ok, as reconcile will go into - // an exponential backoff - resp, err := m.NetworkLoadBalancerClient.DeleteBackend(ctx, networkloadbalancer.DeleteBackendRequest{ - NetworkLoadBalancerId: loadbalancerId, - BackendSetName: backendSet.Name, - BackendName: common.String(m.Name()), - }) - if err != nil { - return err + desiredBackendSetNames := m.desiredAPIServerBackendSetNames() + for backendSetName, backendSet := range lb.BackendSets { + if backendSet.Name == nil { + backendSet.Name = common.String(backendSetName) } - m.OCIMachine.Status.DeleteBackendWorkRequestId = *resp.OpcWorkRequestId - logger.Info("Delete instance from LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) - logger.Info("Waiting for LB work request to be complete") - _, err = ociutil.AwaitNLBWorkRequest(ctx, m.NetworkLoadBalancerClient, resp.OpcWorkRequestId) - if err != nil { - return err + desiredBackendName := m.desiredNLBBackendName(backendSetName, desiredBackendSetNames) + legacyBackendName := m.Name() + backendNameToDelete := "" + if m.containsNLBBackend(backendSet, desiredBackendName) { + backendNameToDelete = desiredBackendName + } else if desiredBackendName != legacyBackendName && m.containsNLBBackend(backendSet, legacyBackendName) { + backendNameToDelete = legacyBackendName + } + if backendNameToDelete != "" { + logger := m.Logger.WithValues("backend-set", *backendSet.Name) + // we always try to delete the backend if it exists during a reconcile loop and wait for the work request + // to complete. If there is a work request in progress, in the rare case, CAPOCI pod restarts during the + // work request, the delete backend call may throw an error which is ok, as reconcile will go into + // an exponential backoff + resp, err := m.NetworkLoadBalancerClient.DeleteBackend(ctx, networkloadbalancer.DeleteBackendRequest{ + NetworkLoadBalancerId: loadbalancerId, + BackendSetName: backendSet.Name, + BackendName: common.String(backendNameToDelete), + }) + if err != nil { + return err + } + m.OCIMachine.Status.DeleteBackendWorkRequestId = *resp.OpcWorkRequestId + logger.Info("Delete instance from LB backend-set", "WorkRequestId", resp.OpcWorkRequestId) + logger.Info("Waiting for LB work request to be complete") + _, err = ociutil.AwaitNLBWorkRequest(ctx, m.NetworkLoadBalancerClient, resp.OpcWorkRequestId) + if err != nil { + return err + } } } } return nil } +func (m *MachineScope) desiredAPIServerBackendSetNames() []string { + backendSets := m.OCIClusterAccessor.GetNetworkSpec().APIServerLB.CanonicalAPIServerBackendSets() + names := make([]string, 0, len(backendSets)) + for _, backendSet := range backendSets { + names = append(names, backendSet.Name) + } + return names +} + +func (m *MachineScope) desiredAPIServerBackendPort(backendSetName string) int32 { + defaultPort := m.OCIClusterAccessor.GetControlPlaneEndpoint().Port + for _, backendSet := range m.OCIClusterAccessor.GetNetworkSpec().APIServerLB.CanonicalAPIServerBackendSets() { + if backendSet.Name != backendSetName { + continue + } + return desiredAPIServerListenerPort(defaultPort, backendSet) + } + return defaultPort +} + +func (m *MachineScope) createBackendRetryToken(backendSetName string, backendPort int32) *string { + uid := string(m.OCIMachine.UID) + if len(m.desiredAPIServerBackendSetNames()) <= 1 { + return ociutil.GetOPCRetryToken("%s-%s", "create-backend", uid) + } + scope := fmt.Sprintf("%s:%d", backendSetName, backendPort) + return ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", uid, stableShortHash(scope)) +} + +func (m *MachineScope) desiredLBBackendName(instanceIP, backendSetName string) string { + backendPort := m.desiredAPIServerBackendPort(backendSetName) + return instanceIP + ":" + strconv.Itoa(int(backendPort)) +} + +func (m *MachineScope) desiredNLBBackendName(backendSetName string, desiredBackendSetNames []string) string { + // Preserve current backend naming for all single-backend-set configurations. + if len(desiredBackendSetNames) <= 1 { + return m.Name() + } + // Preserve legacy name for the primary/default API server backend set. + if backendSetName == APIServerLBBackendSetName { + return m.Name() + } + return fmt.Sprintf("%s-%s", m.Name(), stableShortHash(backendSetName)) +} + func (m *MachineScope) containsNLBBackend(backendSet networkloadbalancer.BackendSet, backendName string) bool { for _, backend := range backendSet.Backends { if ptr.StringEquals(backend.Name, backendName) { diff --git a/cloud/scope/machine_backendsets_test.go b/cloud/scope/machine_backendsets_test.go new file mode 100644 index 000000000..34b935c43 --- /dev/null +++ b/cloud/scope/machine_backendsets_test.go @@ -0,0 +1,292 @@ +package scope + +import ( + "context" + "fmt" + "testing" + + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer/mock_nlb" + "github.com/oracle/cluster-api-provider-oci/cloud/services/workrequests/mock_workrequests" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestNLBCreateMembershipAcrossBackendSets(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + nlbClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + wrClient := mock_workrequests.NewMockClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{UID: "uid"}, + Spec: infrastructurev1beta2.OCIClusterSpec{ + NetworkSpec: infrastructurev1beta2.NetworkSpec{ + APIServerLB: infrastructurev1beta2.LoadBalancer{ + LoadBalancerId: common.String("nlbid"), + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: "set-a"}, + {Name: "set-b"}, + }, + }, + }, + }, + }, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + + ms, err := NewMachineScope(MachineScopeParams{ + NetworkLoadBalancerClient: nlbClient, + WorkRequestsClient: wrClient, + OCIMachine: &infrastructurev1beta2.OCIMachine{ + ObjectMeta: metav1.ObjectMeta{Name: "test", UID: "uid"}, + }, + Machine: &clusterv1.Machine{}, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{ + OCICluster: ociCluster, + }, + Client: client, + }) + g.Expect(err).To(BeNil()) + ms.OCIMachine.Status.Addresses = []clusterv1beta1.MachineAddress{{ + Type: clusterv1beta1.MachineInternalIP, + Address: "1.1.1.1", + }} + + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + BackendSets: map[string]networkloadbalancer.BackendSet{ + "set-a": {Name: common.String("set-a"), Backends: []networkloadbalancer.Backend{}}, + "set-b": {Name: common.String("set-b"), Backends: []networkloadbalancer.Backend{}}, + }, + }, + }, nil) + + nlbClient.EXPECT().CreateBackend(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + BackendSetName: common.String("set-a"), + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String("1.1.1.1"), + Port: common.Int(6443), + Name: common.String(fmt.Sprintf("test-%s", stableShortHash("set-a"))), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("set-a:6443")), + })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-a")}, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wrid-a"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil) + + nlbClient.EXPECT().CreateBackend(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + BackendSetName: common.String("set-b"), + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String("1.1.1.1"), + Port: common.Int(6443), + Name: common.String(fmt.Sprintf("test-%s", stableShortHash("set-b"))), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("set-b:6443")), + })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-b")}, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wrid-b"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil) + + g.Expect(ms.ReconcileCreateInstanceOnLB(context.Background())).To(Succeed()) +} + +func TestNLBDeleteMembershipFromAllBackendSets(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + nlbClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + wrClient := mock_workrequests.NewMockClient(mockCtrl) + client := fake.NewClientBuilder().Build() + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{UID: "uid"}, + Spec: infrastructurev1beta2.OCIClusterSpec{ + NetworkSpec: infrastructurev1beta2.NetworkSpec{ + APIServerLB: infrastructurev1beta2.LoadBalancer{ + LoadBalancerId: common.String("nlbid"), + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: "set-a"}, + }, + }, + }, + }, + }, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + + ms, err := NewMachineScope(MachineScopeParams{ + NetworkLoadBalancerClient: nlbClient, + WorkRequestsClient: wrClient, + OCIMachine: &infrastructurev1beta2.OCIMachine{ + ObjectMeta: metav1.ObjectMeta{Name: "test", UID: "uid"}, + }, + Machine: &clusterv1.Machine{}, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{ + OCICluster: ociCluster, + }, + Client: client, + }) + g.Expect(err).To(BeNil()) + + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + BackendSets: map[string]networkloadbalancer.BackendSet{ + "set-a": {Name: common.String("set-a"), Backends: []networkloadbalancer.Backend{{Name: common.String("test")}}}, + "set-old": {Name: common.String("set-old"), Backends: []networkloadbalancer.Backend{{Name: common.String("test")}}}, + }, + }, + }, nil) + + seen := map[string]bool{} + nlbClient.EXPECT().DeleteBackend(gomock.Any(), gomock.Any()).Times(2).DoAndReturn( + func(_ context.Context, req networkloadbalancer.DeleteBackendRequest) (networkloadbalancer.DeleteBackendResponse, error) { + if req.NetworkLoadBalancerId == nil || *req.NetworkLoadBalancerId != "nlbid" { + t.Fatalf("unexpected load balancer id: %#v", req.NetworkLoadBalancerId) + } + if req.BackendName == nil || *req.BackendName != "test" { + t.Fatalf("unexpected backend name: %#v", req.BackendName) + } + if req.BackendSetName == nil { + t.Fatalf("backend set name is nil") + } + name := *req.BackendSetName + if name != "set-a" && name != "set-old" { + t.Fatalf("unexpected backend set name: %q", name) + } + seen[name] = true + return networkloadbalancer.DeleteBackendResponse{ + OpcWorkRequestId: common.String("wrid-" + name), + }, nil + }, + ) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Any()).Times(2).DoAndReturn( + func(_ context.Context, req networkloadbalancer.GetWorkRequestRequest) (networkloadbalancer.GetWorkRequestResponse, error) { + if req.WorkRequestId == nil { + t.Fatalf("work request id is nil") + } + return networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil + }, + ) + + g.Expect(ms.ReconcileDeleteInstanceOnLB(context.Background())).To(Succeed()) + g.Expect(seen["set-a"]).To(BeTrue()) + g.Expect(seen["set-old"]).To(BeTrue()) +} + +func TestNLBCreateMembership_DefaultAndSecondaryBackendSetNaming(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + nlbClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + wrClient := mock_workrequests.NewMockClient(mockCtrl) + client := fake.NewClientBuilder().Build() + secondaryPort := int32(9345) + ociCluster := &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{UID: "uid"}, + Spec: infrastructurev1beta2.OCIClusterSpec{ + NetworkSpec: infrastructurev1beta2.NetworkSpec{ + APIServerLB: infrastructurev1beta2.LoadBalancer{ + LoadBalancerId: common.String("nlbid"), + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "apiserver-lb-backendset-2", ListenerPort: &secondaryPort}, + }, + }, + }, + }, + }, + } + ociCluster.Spec.ControlPlaneEndpoint.Port = 6443 + + ms, err := NewMachineScope(MachineScopeParams{ + NetworkLoadBalancerClient: nlbClient, + WorkRequestsClient: wrClient, + OCIMachine: &infrastructurev1beta2.OCIMachine{ + ObjectMeta: metav1.ObjectMeta{Name: "test", UID: "uid"}, + }, + Machine: &clusterv1.Machine{}, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: OCISelfManagedCluster{ + OCICluster: ociCluster, + }, + Client: client, + }) + g.Expect(err).To(BeNil()) + ms.OCIMachine.Status.Addresses = []clusterv1beta1.MachineAddress{{ + Type: clusterv1beta1.MachineInternalIP, + Address: "1.1.1.1", + }} + + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: {Name: common.String(APIServerLBBackendSetName), Backends: []networkloadbalancer.Backend{}}, + "apiserver-lb-backendset-2": {Name: common.String("apiserver-lb-backendset-2"), Backends: []networkloadbalancer.Backend{}}, + }, + }, + }, nil) + + nlbClient.EXPECT().CreateBackend(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + BackendSetName: common.String(APIServerLBBackendSetName), + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String("1.1.1.1"), + Port: common.Int(6443), + Name: common.String("test"), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash(APIServerLBBackendSetName+":6443")), + })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-default")}, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wrid-default"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil) + + nlbClient.EXPECT().CreateBackend(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendRequest{ + NetworkLoadBalancerId: common.String("nlbid"), + BackendSetName: common.String("apiserver-lb-backendset-2"), + CreateBackendDetails: networkloadbalancer.CreateBackendDetails{ + IpAddress: common.String("1.1.1.1"), + Port: common.Int(9345), + Name: common.String(fmt.Sprintf("test-%s", stableShortHash("apiserver-lb-backendset-2"))), + }, + OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s-%s", "create-backend", "uid", stableShortHash("apiserver-lb-backendset-2:9345")), + })).Return(networkloadbalancer.CreateBackendResponse{OpcWorkRequestId: common.String("wrid-secondary")}, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wrid-secondary"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil) + + g.Expect(ms.ReconcileCreateInstanceOnLB(context.Background())).To(Succeed()) +} diff --git a/cloud/scope/named_resource_reconcile_helpers.go b/cloud/scope/named_resource_reconcile_helpers.go new file mode 100644 index 000000000..d0210ce4a --- /dev/null +++ b/cloud/scope/named_resource_reconcile_helpers.go @@ -0,0 +1,67 @@ +package scope + +import infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" + +func desiredAPIServerBackendSets(lb infrastructurev1beta2.LoadBalancer) []infrastructurev1beta2.NLBBackendSet { + canonicalBackendSets := lb.CanonicalAPIServerBackendSets() + if len(canonicalBackendSets) > 0 { + return canonicalBackendSets + } + + return []infrastructurev1beta2.NLBBackendSet{{Name: APIServerLBBackendSetName}} +} + +func reconcileNamedResources[Actual any, Desired any]( + actual map[string]Actual, + desired map[string]Desired, + create func(name string, desired Desired) error, + needsUpdate func(actual Actual, desired Desired) bool, + update func(name string, desired Desired) error, +) error { + for name, desiredResource := range desired { + actualResource, exists := actual[name] + if !exists { + if err := create(name, desiredResource); err != nil { + return err + } + continue + } + if !needsUpdate(actualResource, desiredResource) { + continue + } + if err := update(name, desiredResource); err != nil { + return err + } + } + return nil +} + +func deleteStaleNamedResources[Actual any, Desired any]( + actual map[string]Actual, + desired map[string]Desired, + shouldDelete func(actual Actual) bool, + deleteFn func(name string, actual Actual) error, +) (bool, error) { + deleted := false + for name, actualResource := range actual { + if _, keep := desired[name]; keep { + continue + } + if shouldDelete != nil && !shouldDelete(actualResource) { + continue + } + if err := deleteFn(name, actualResource); err != nil { + return deleted, err + } + deleted = true + } + return deleted, nil +} + +func desiredNameSet[T any](resources map[string]T) map[string]struct{} { + names := make(map[string]struct{}, len(resources)) + for name := range resources { + names[name] = struct{}{} + } + return names +} diff --git a/cloud/scope/network_load_balancer_reconciler.go b/cloud/scope/network_load_balancer_reconciler.go index 9736a5122..ca0456e08 100644 --- a/cloud/scope/network_load_balancer_reconciler.go +++ b/cloud/scope/network_load_balancer_reconciler.go @@ -31,7 +31,7 @@ import ( // ReconcileApiServerNLB tries to move the Network Load Balancer to the desired OCICluster Spec func (s *ClusterScope) ReconcileApiServerNLB(ctx context.Context) error { - desiredApiServerNLB := s.NLBSpec() + desiredApiServerNLB := s.DesiredAPIServerNetworkLoadBalancer() nlb, err := s.GetNetworkLoadBalancers(ctx) if err != nil { @@ -97,13 +97,12 @@ func (s *ClusterScope) DeleteApiServerNLB(ctx context.Context) error { return nil } -// NLBSpec builds the Network LoadBalancer from the ClusterScope and returns it -func (s *ClusterScope) NLBSpec() infrastructurev1beta2.LoadBalancer { - nlbSpec := infrastructurev1beta2.LoadBalancer{ - Name: s.GetControlPlaneLoadBalancerName(), - NLBSpec: s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec, - } - return nlbSpec +// DesiredAPIServerNetworkLoadBalancer builds the desired network load balancer from the ClusterScope. +func (s *ClusterScope) DesiredAPIServerNetworkLoadBalancer() infrastructurev1beta2.LoadBalancer { + return infrastructurev1beta2.NewAPIServerLoadBalancer( + s.GetControlPlaneLoadBalancerName(), + *s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.APIServerLoadBalancerSpec(), + ) } // GetControlPlaneLoadBalancerName returns the user defined APIServerLB name from the spec or @@ -134,6 +133,170 @@ func (s *ClusterScope) UpdateNLB(ctx context.Context, nlb infrastructurev1beta2. s.Logger.Error(err, "failed to reconcile the apiserver NLB, failed to update nlb") return errors.Wrap(err, "failed to reconcile the apiserver NLB, failed to update nlb") } + if err := s.reconcileNLBResources(ctx, nlb); err != nil { + return err + } + return nil +} + +func (s *ClusterScope) reconcileNLBResources(ctx context.Context, nlb infrastructurev1beta2.LoadBalancer) error { + nlbID := s.OCIClusterAccessor.GetNetworkSpec().APIServerLB.LoadBalancerId + actualResp, err := s.NetworkLoadBalancerClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: nlbID, + }) + if err != nil { + return errors.Wrap(err, "failed to get nlb for resource reconciliation") + } + desiredListeners, desiredBackendSets := s.buildDesiredNLBListenersAndBackendSets(nlb) + + if err := reconcileNamedResources( + actualResp.NetworkLoadBalancer.Listeners, + desiredListeners, + func(name string, desired networkloadbalancer.ListenerDetails) error { + resp, err := s.NetworkLoadBalancerClient.CreateListener(ctx, networkloadbalancer.CreateListenerRequest{ + NetworkLoadBalancerId: nlbID, + CreateListenerDetails: networkloadbalancer.CreateListenerDetails{ + Name: common.String(name), + DefaultBackendSetName: desired.DefaultBackendSetName, + Port: desired.Port, + Protocol: desired.Protocol, + }, + }) + if err != nil { + if isAlreadyExistsOCIError(err) { + s.Logger.Info("NLB listener already exists during reconcile; continuing", "listener", name) + return nil + } + return errors.Wrapf(err, "failed to create nlb listener %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting create listener %q", name) + } + return nil + }, + nlbListenerNeedsUpdate, + func(name string, desired networkloadbalancer.ListenerDetails) error { + resp, err := s.NetworkLoadBalancerClient.UpdateListener(ctx, networkloadbalancer.UpdateListenerRequest{ + NetworkLoadBalancerId: nlbID, + ListenerName: common.String(name), + UpdateListenerDetails: networkloadbalancer.UpdateListenerDetails{ + DefaultBackendSetName: desired.DefaultBackendSetName, + Port: desired.Port, + Protocol: desired.Protocol, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update nlb listener %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update listener %q", name) + } + return nil + }, + ); err != nil { + return err + } + staleListenerDeleted, err := deleteStaleNamedResources( + actualResp.NetworkLoadBalancer.Listeners, + desiredNameSet(desiredListeners), + func(networkloadbalancer.Listener) bool { return true }, + func(name string, _ networkloadbalancer.Listener) error { + resp, err := s.NetworkLoadBalancerClient.DeleteListener(ctx, networkloadbalancer.DeleteListenerRequest{ + NetworkLoadBalancerId: nlbID, + ListenerName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale nlb listener %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete listener %q", name) + } + return nil + }, + ) + if err != nil { + return err + } + if staleListenerDeleted { + actualResp, err = s.NetworkLoadBalancerClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: nlbID, + }) + if err != nil { + return errors.Wrap(err, "failed to refresh nlb after stale listener reconciliation") + } + } + + if err := reconcileNamedResources( + actualResp.NetworkLoadBalancer.BackendSets, + desiredBackendSets, + func(name string, desired networkloadbalancer.BackendSetDetails) error { + resp, err := s.NetworkLoadBalancerClient.CreateBackendSet(ctx, networkloadbalancer.CreateBackendSetRequest{ + NetworkLoadBalancerId: nlbID, + CreateBackendSetDetails: networkloadbalancer.CreateBackendSetDetails{ + Name: common.String(name), + Policy: desired.Policy, + HealthChecker: nlbHealthCheckerToDetails(desired.HealthChecker), + IsPreserveSource: desired.IsPreserveSource, + IsFailOpen: desired.IsFailOpen, + IsInstantFailoverEnabled: desired.IsInstantFailoverEnabled, + }, + }) + if err != nil { + if isAlreadyExistsOCIError(err) { + s.Logger.Info("NLB backend set already exists during reconcile; continuing", "backendSet", name) + return nil + } + return errors.Wrapf(err, "failed to create nlb backend set %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting create backend set %q", name) + } + return nil + }, + nlbBackendSetNeedsUpdate, + func(name string, desired networkloadbalancer.BackendSetDetails) error { + resp, err := s.NetworkLoadBalancerClient.UpdateBackendSet(ctx, networkloadbalancer.UpdateBackendSetRequest{ + NetworkLoadBalancerId: nlbID, + BackendSetName: common.String(name), + UpdateBackendSetDetails: networkloadbalancer.UpdateBackendSetDetails{ + Policy: common.String(string(desired.Policy)), + HealthChecker: nlbHealthCheckerToDetails(desired.HealthChecker), + IsPreserveSource: desired.IsPreserveSource, + IsFailOpen: desired.IsFailOpen, + IsInstantFailoverEnabled: desired.IsInstantFailoverEnabled, + }, + }) + if err != nil { + return errors.Wrapf(err, "failed to update nlb backend set %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting update backend set %q", name) + } + return nil + }, + ); err != nil { + return err + } + if _, err := deleteStaleNamedResources( + actualResp.NetworkLoadBalancer.BackendSets, + desiredNameSet(desiredBackendSets), + func(actual networkloadbalancer.BackendSet) bool { return len(actual.Backends) == 0 }, + func(name string, _ networkloadbalancer.BackendSet) error { + resp, err := s.NetworkLoadBalancerClient.DeleteBackendSet(ctx, networkloadbalancer.DeleteBackendSetRequest{ + NetworkLoadBalancerId: nlbID, + BackendSetName: common.String(name), + }) + if err != nil { + return errors.Wrapf(err, "failed to delete stale nlb backend set %q", name) + } + if _, err := ociutil.AwaitNLBWorkRequest(ctx, s.NetworkLoadBalancerClient, resp.OpcWorkRequestId); err != nil { + return errors.Wrapf(err, "failed awaiting delete backend set %q", name) + } + return nil + }, + ); err != nil { + return err + } return nil } @@ -144,36 +307,7 @@ func (s *ClusterScope) UpdateNLB(ctx context.Context, nlb infrastructurev1beta2. // See https://docs.oracle.com/en-us/iaas/Content/NetworkLoadBalancer/overview.htm for more details on the Network // Load Balancer func (s *ClusterScope) CreateNLB(ctx context.Context, lb infrastructurev1beta2.LoadBalancer) (*string, *string, error) { - isPreserverSourceIp := lb.NLBSpec.BackendSetDetails.IsPreserveSource - if isPreserverSourceIp == nil { - isPreserverSourceIp = common.Bool(false) - } - listenerDetails := make(map[string]networkloadbalancer.ListenerDetails) - listenerDetails[APIServerLBListener] = networkloadbalancer.ListenerDetails{ - Protocol: networkloadbalancer.ListenerProtocolsTcp, - Port: common.Int(int(s.APIServerPort())), - DefaultBackendSetName: common.String(APIServerLBBackendSetName), - Name: common.String(APIServerLBListener), - } - - backendSetDetails := make(map[string]networkloadbalancer.BackendSetDetails) - healthCheckUrl := lb.NLBSpec.BackendSetDetails.HealthChecker.UrlPath - if healthCheckUrl == nil { - healthCheckUrl = common.String("/healthz") - } - backendSetDetails[APIServerLBBackendSetName] = networkloadbalancer.BackendSetDetails{ - Policy: LoadBalancerPolicy, - IsPreserveSource: isPreserverSourceIp, - IsFailOpen: lb.NLBSpec.BackendSetDetails.IsFailOpen, - IsInstantFailoverEnabled: lb.NLBSpec.BackendSetDetails.IsInstantFailoverEnabled, - HealthChecker: &networkloadbalancer.HealthChecker{ - Port: common.Int(int(s.APIServerPort())), - Protocol: networkloadbalancer.HealthCheckProtocolsHttps, - UrlPath: healthCheckUrl, - ReturnCode: common.Int(200), - }, - Backends: []networkloadbalancer.Backend{}, - } + listenerDetails, backendSetDetails := s.buildDesiredNLBListenersAndBackendSets(lb) var controlPlaneEndpointSubnets []string for _, subnet := range ptr.ToSubnetSlice(s.OCIClusterAccessor.GetNetworkSpec().Vcn.Subnets) { @@ -184,9 +318,9 @@ func (s *ClusterScope) CreateNLB(ctx context.Context, lb infrastructurev1beta2.L } } var reservedIps []networkloadbalancer.ReservedIp - if len(lb.NLBSpec.ReservedIpIds) > 0 { + if len(lb.APIServerLoadBalancerSpec().ReservedIpIds) > 0 { // since max is one we only take the first ip id supplied - reservedIps = append(reservedIps, networkloadbalancer.ReservedIp{Id: common.String(lb.NLBSpec.ReservedIpIds[0])}) + reservedIps = append(reservedIps, networkloadbalancer.ReservedIp{Id: common.String(lb.APIServerLoadBalancerSpec().ReservedIpIds[0])}) } if len(controlPlaneEndpointSubnets) < 1 { @@ -248,6 +382,80 @@ func (s *ClusterScope) CreateNLB(ctx context.Context, lb infrastructurev1beta2.L return nlb.Id, nlbIp, nil } +func (s *ClusterScope) buildDesiredNLBListenersAndBackendSets(lb infrastructurev1beta2.LoadBalancer) (map[string]networkloadbalancer.ListenerDetails, map[string]networkloadbalancer.BackendSetDetails) { + canonicalBackendSets := desiredAPIServerBackendSets(lb) + + backendSetDetails := make(map[string]networkloadbalancer.BackendSetDetails, len(canonicalBackendSets)) + for _, backendSet := range canonicalBackendSets { + isPreserveSourceIp := backendSet.BackendSetDetails.IsPreserveSource + if isPreserveSourceIp == nil { + isPreserveSourceIp = common.Bool(false) + } + healthCheckURL := backendSet.BackendSetDetails.HealthChecker.UrlPath + if healthCheckURL == nil { + healthCheckURL = common.String("/healthz") + } + backendSetDetails[backendSet.Name] = networkloadbalancer.BackendSetDetails{ + Policy: LoadBalancerPolicy, + IsPreserveSource: isPreserveSourceIp, + IsFailOpen: backendSet.BackendSetDetails.IsFailOpen, + IsInstantFailoverEnabled: backendSet.BackendSetDetails.IsInstantFailoverEnabled, + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(int(s.APIServerPort())), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: healthCheckURL, + ReturnCode: common.Int(200), + }, + Backends: []networkloadbalancer.Backend{}, + } + } + + listenerDetails := make(map[string]networkloadbalancer.ListenerDetails, len(canonicalBackendSets)) + for i, backendSet := range canonicalBackendSets { + listenerName := desiredAPIServerListenerName(i, len(canonicalBackendSets), backendSet.Name) + port := desiredAPIServerListenerPort(s.APIServerPort(), backendSet) + listenerDetails[listenerName] = networkloadbalancer.ListenerDetails{ + Protocol: networkloadbalancer.ListenerProtocolsTcp, + Port: common.Int(int(port)), + DefaultBackendSetName: common.String(backendSet.Name), + Name: common.String(listenerName), + } + } + return listenerDetails, backendSetDetails +} + +func boolPtrEqual(a, b *bool) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +func intPtrEqual(a, b *int) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +func nlbHealthCheckerToDetails(in *networkloadbalancer.HealthChecker) *networkloadbalancer.HealthCheckerDetails { + if in == nil { + return nil + } + return &networkloadbalancer.HealthCheckerDetails{ + Protocol: in.Protocol, + Port: in.Port, + Retries: in.Retries, + TimeoutInMillis: in.TimeoutInMillis, + IntervalInMillis: in.IntervalInMillis, + UrlPath: in.UrlPath, + ResponseBodyRegex: in.ResponseBodyRegex, + ReturnCode: in.ReturnCode, + RequestData: in.RequestData, + ResponseData: in.ResponseData, + } +} + func (s *ClusterScope) getNetworkLoadbalancerIp(nlb networkloadbalancer.NetworkLoadBalancer) (*string, error) { var nlbIp *string if len(nlb.IpAddresses) < 1 { @@ -269,14 +477,61 @@ func (s *ClusterScope) getNetworkLoadbalancerIp(nlb networkloadbalancer.NetworkL } // IsNLBEqual determines if the actual networkloadbalancer.NetworkLoadBalancer is equal to the desired. -// Equality is determined by DisplayName +// Equality is determined by the NLB identity plus desired listener/backend-set resources matching. func (s *ClusterScope) IsNLBEqual(actual *networkloadbalancer.NetworkLoadBalancer, desired infrastructurev1beta2.LoadBalancer) bool { if desired.Name != *actual.DisplayName { return false } + + desiredListeners, desiredBackendSets := s.buildDesiredNLBListenersAndBackendSets(desired) + if len(actual.Listeners) != len(desiredListeners) { + return false + } + for name, desiredListener := range desiredListeners { + actualListener, exists := actual.Listeners[name] + if !exists { + return false + } + if nlbListenerNeedsUpdate(actualListener, desiredListener) { + return false + } + } + + if len(actual.BackendSets) != len(desiredBackendSets) { + return false + } + for name, desiredBackendSet := range desiredBackendSets { + actualBackendSet, exists := actual.BackendSets[name] + if !exists { + return false + } + if nlbBackendSetNeedsUpdate(actualBackendSet, desiredBackendSet) { + return false + } + } + return true } +func nlbListenerNeedsUpdate(actual networkloadbalancer.Listener, desired networkloadbalancer.ListenerDetails) bool { + return !intPtrEqual(actual.Port, desired.Port) || + !ptr.StringEquals(actual.DefaultBackendSetName, ptr.ToString(desired.DefaultBackendSetName)) || + actual.Protocol != desired.Protocol +} + +func nlbBackendSetNeedsUpdate(actual networkloadbalancer.BackendSet, desired networkloadbalancer.BackendSetDetails) bool { + if actual.HealthChecker == nil || desired.HealthChecker == nil { + return true + } + return actual.Policy != desired.Policy || + !boolPtrEqual(actual.IsPreserveSource, desired.IsPreserveSource) || + !boolPtrEqual(actual.IsFailOpen, desired.IsFailOpen) || + !boolPtrEqual(actual.IsInstantFailoverEnabled, desired.IsInstantFailoverEnabled) || + !intPtrEqual(actual.HealthChecker.Port, desired.HealthChecker.Port) || + actual.HealthChecker.Protocol != desired.HealthChecker.Protocol || + !ptr.StringEquals(actual.HealthChecker.UrlPath, ptr.ToString(desired.HealthChecker.UrlPath)) +} + // GetNetworkLoadBalancers retrieves the Cluster's networkloadbalancer.NetworkLoadBalancer using the one of the following methods // // 1. the OCICluster's spec LoadBalancerId diff --git a/cloud/scope/network_load_balancer_reconciler_test.go b/cloud/scope/network_load_balancer_reconciler_test.go index 049a7b315..231521de8 100644 --- a/cloud/scope/network_load_balancer_reconciler_test.go +++ b/cloud/scope/network_load_balancer_reconciler_test.go @@ -35,6 +35,40 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +type extendedMockNLBClient struct { + *mock_nlb.MockNetworkLoadBalancerClient + createBackendSetFn func(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) + updateBackendSetFn func(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) + deleteBackendSetFn func(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) + createListenerFn func(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) + updateListenerFn func(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) + deleteListenerFn func(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) +} + +func (c *extendedMockNLBClient) CreateBackendSet(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) { + return c.createBackendSetFn(ctx, request) +} + +func (c *extendedMockNLBClient) UpdateBackendSet(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) { + return c.updateBackendSetFn(ctx, request) +} + +func (c *extendedMockNLBClient) DeleteBackendSet(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) { + return c.deleteBackendSetFn(ctx, request) +} + +func (c *extendedMockNLBClient) CreateListener(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) { + return c.createListenerFn(ctx, request) +} + +func (c *extendedMockNLBClient) UpdateListener(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) { + return c.updateListenerFn(ctx, request) +} + +func (c *extendedMockNLBClient) DeleteListener(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) { + return c.deleteListenerFn(ctx, request) +} + func TestNLBReconciliation(t *testing.T) { var ( cs *ClusterScope @@ -109,10 +143,136 @@ func TestNLBReconciliation(t *testing.T) { IsPublic: common.Bool(true), }, }, + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + }, }, }, nil) }, }, + { + name: "nlb spec-only resource change triggers update", + errorExpected: false, + testSpecificSetup: func(clusterScope *ClusterScope, nlbClient *mock_nlb.MockNetworkLoadBalancerClient) { + secondaryPort := int32(9345) + secondaryListenerName := desiredAPIServerListenerName(1, 2, "rollout-set") + clusterScope.OCIClusterAccessor.GetNetworkSpec().APIServerLB.LoadBalancerId = common.String("nlb-id") + clusterScope.OCIClusterAccessor.GetNetworkSpec().APIServerLB.NLBSpec.BackendSets = []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "rollout-set", ListenerPort: &secondaryPort}, + } + getReq := networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + } + getResp := networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + LifecycleState: networkloadbalancer.LifecycleStateActive, + Id: common.String("nlb-id"), + FreeformTags: tags, + DefinedTags: make(map[string]map[string]interface{}), + IsPrivate: common.Bool(false), + DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + IpAddresses: []networkloadbalancer.IpAddress{ + { + IpAddress: common.String("2.2.2.2"), + IsPublic: common.Bool(true), + }, + }, + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + }, + }, + } + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(getReq)).Return(getResp, nil) + nlbClient.EXPECT().UpdateNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.UpdateNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + UpdateNetworkLoadBalancerDetails: networkloadbalancer.UpdateNetworkLoadBalancerDetails{ + DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")), + }, + })). + Return(networkloadbalancer.UpdateNetworkLoadBalancerResponse{ + OpcWorkRequestId: common.String("wr-nlb-update"), + }, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-nlb-update"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{ + Status: networkloadbalancer.OperationStatusSucceeded, + }, + }, nil) + nlbClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(getReq)).Return(getResp, nil) + nlbClient.EXPECT().CreateListener(gomock.Any(), gomock.Eq(networkloadbalancer.CreateListenerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + CreateListenerDetails: networkloadbalancer.CreateListenerDetails{ + Name: common.String(secondaryListenerName), + DefaultBackendSetName: common.String("rollout-set"), + Port: common.Int(int(secondaryPort)), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + })).Return(networkloadbalancer.CreateListenerResponse{ + OpcWorkRequestId: common.String("wr-create-listener"), + }, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-create-listener"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{ + Status: networkloadbalancer.OperationStatusSucceeded, + }, + }, nil) + nlbClient.EXPECT().CreateBackendSet(gomock.Any(), gomock.Eq(networkloadbalancer.CreateBackendSetRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + CreateBackendSetDetails: networkloadbalancer.CreateBackendSetDetails{ + Name: common.String("rollout-set"), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthCheckerDetails{Port: common.Int(6443), Protocol: networkloadbalancer.HealthCheckProtocolsHttps, UrlPath: common.String("/healthz"), ReturnCode: common.Int(200)}, + }, + })).Return(networkloadbalancer.CreateBackendSetResponse{ + OpcWorkRequestId: common.String("wr-create-backendset"), + }, nil) + nlbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-create-backendset"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{ + Status: networkloadbalancer.OperationStatusSucceeded, + }, + }, nil) + }, + }, { name: "nlb does not have ip address", errorExpected: true, @@ -838,6 +998,249 @@ func TestNLBReconciliation(t *testing.T) { } } +func TestReconcileNLBResources_IgnoresAlreadyExistsOnCreate(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + baseClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + nlbClient := &extendedMockNLBClient{ + MockNetworkLoadBalancerClient: baseClient, + createBackendSetFn: func(_ context.Context, request networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) { + if request.CreateBackendSetDetails.Name == nil || *request.CreateBackendSetDetails.Name != "rollout-set" { + t.Fatalf("unexpected backend set create request: %#v", request) + } + return networkloadbalancer.CreateBackendSetResponse{}, mockServiceError{ + status: 409, + code: "Conflict", + message: "backend set already exists", + } + }, + updateBackendSetFn: func(_ context.Context, _ networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) { + t.Fatalf("unexpected backend set update") + return networkloadbalancer.UpdateBackendSetResponse{}, nil + }, + deleteBackendSetFn: func(_ context.Context, _ networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) { + t.Fatalf("unexpected backend set delete") + return networkloadbalancer.DeleteBackendSetResponse{}, nil + }, + createListenerFn: func(_ context.Context, request networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) { + if request.CreateListenerDetails.Name == nil || *request.CreateListenerDetails.Name != desiredAPIServerListenerName(1, 2, "rollout-set") { + t.Fatalf("unexpected listener create request: %#v", request) + } + return networkloadbalancer.CreateListenerResponse{}, mockServiceError{ + status: 409, + code: "Conflict", + message: "listener already exists", + } + }, + updateListenerFn: func(_ context.Context, _ networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) { + t.Fatalf("unexpected listener update") + return networkloadbalancer.UpdateListenerResponse{}, nil + }, + deleteListenerFn: func(_ context.Context, _ networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) { + t.Fatalf("unexpected listener delete") + return networkloadbalancer.DeleteListenerResponse{}, nil + }, + } + + client := fake.NewClientBuilder().Build() + ociClusterAccessor := OCISelfManagedCluster{ + &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + }, + } + ociClusterAccessor.OCICluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociClusterAccessor.OCICluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("nlb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + NetworkLoadBalancerClient: nlbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: ociClusterAccessor, + Client: client, + }) + g.Expect(err).To(BeNil()) + + secondaryPort := int32(9345) + desiredNLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + {Name: "rollout-set", ListenerPort: &secondaryPort}, + }, + }, + } + + baseClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + }, + }, + }, nil) + + g.Expect(cs.reconcileNLBResources(context.Background(), desiredNLB)).To(Succeed()) +} + +func TestReconcileNLBResources_RefreshesBeforeDeletingStaleBackendSets(t *testing.T) { + g := NewWithT(t) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + baseClient := mock_nlb.NewMockNetworkLoadBalancerClient(mockCtrl) + nlbClient := &extendedMockNLBClient{ + MockNetworkLoadBalancerClient: baseClient, + createBackendSetFn: func(_ context.Context, _ networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) { + t.Fatalf("unexpected backend set create") + return networkloadbalancer.CreateBackendSetResponse{}, nil + }, + updateBackendSetFn: func(_ context.Context, _ networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) { + t.Fatalf("unexpected backend set update") + return networkloadbalancer.UpdateBackendSetResponse{}, nil + }, + deleteBackendSetFn: func(_ context.Context, _ networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) { + t.Fatalf("unexpected backend set delete") + return networkloadbalancer.DeleteBackendSetResponse{}, nil + }, + createListenerFn: func(_ context.Context, _ networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) { + t.Fatalf("unexpected listener create") + return networkloadbalancer.CreateListenerResponse{}, nil + }, + updateListenerFn: func(_ context.Context, _ networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) { + t.Fatalf("unexpected listener update") + return networkloadbalancer.UpdateListenerResponse{}, nil + }, + deleteListenerFn: func(_ context.Context, request networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) { + if request.ListenerName == nil || *request.ListenerName != "stale-listener" { + t.Fatalf("unexpected listener delete request: %#v", request) + } + return networkloadbalancer.DeleteListenerResponse{OpcWorkRequestId: common.String("wr-delete-listener")}, nil + }, + } + + client := fake.NewClientBuilder().Build() + ociClusterAccessor := OCISelfManagedCluster{ + &infrastructurev1beta2.OCICluster{ + ObjectMeta: metav1.ObjectMeta{ + UID: "a", + Name: "cluster", + }, + Spec: infrastructurev1beta2.OCIClusterSpec{}, + }, + } + ociClusterAccessor.OCICluster.Spec.ControlPlaneEndpoint.Port = 6443 + ociClusterAccessor.OCICluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId = common.String("nlb-id") + + cs, err := NewClusterScope(ClusterScopeParams{ + NetworkLoadBalancerClient: nlbClient, + Cluster: &clusterv1.Cluster{}, + OCIClusterAccessor: ociClusterAccessor, + Client: client, + }) + g.Expect(err).To(BeNil()) + + desiredNLB := infrastructurev1beta2.LoadBalancer{ + NLBSpec: infrastructurev1beta2.NLBSpec{ + BackendSets: []infrastructurev1beta2.NLBBackendSet{ + {Name: APIServerLBBackendSetName}, + }, + }, + } + + gomock.InOrder( + baseClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + "stale-listener": {Name: common.String("stale-listener")}, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + "stale-set": {Name: common.String("stale-set"), Backends: []networkloadbalancer.Backend{}}, + }, + }, + }, nil), + baseClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(networkloadbalancer.GetWorkRequestRequest{ + WorkRequestId: common.String("wr-delete-listener"), + })).Return(networkloadbalancer.GetWorkRequestResponse{ + WorkRequest: networkloadbalancer.WorkRequest{Status: networkloadbalancer.OperationStatusSucceeded}, + }, nil), + baseClient.EXPECT().GetNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String("nlb-id"), + })).Return(networkloadbalancer.GetNetworkLoadBalancerResponse{ + NetworkLoadBalancer: networkloadbalancer.NetworkLoadBalancer{ + Listeners: map[string]networkloadbalancer.Listener{ + APIServerLBListener: { + Name: common.String(APIServerLBListener), + DefaultBackendSetName: common.String(APIServerLBBackendSetName), + Port: common.Int(6443), + Protocol: networkloadbalancer.ListenerProtocolsTcp, + }, + }, + BackendSets: map[string]networkloadbalancer.BackendSet{ + APIServerLBBackendSetName: { + Name: common.String(APIServerLBBackendSetName), + Policy: LoadBalancerPolicy, + IsPreserveSource: common.Bool(false), + HealthChecker: &networkloadbalancer.HealthChecker{ + Port: common.Int(6443), + Protocol: networkloadbalancer.HealthCheckProtocolsHttps, + UrlPath: common.String("/healthz"), + }, + }, + "stale-set": { + Name: common.String("stale-set"), + Backends: []networkloadbalancer.Backend{ + {Name: common.String("backend-1")}, + }, + }, + }, + }, + }, nil), + ) + + g.Expect(cs.reconcileNLBResources(context.Background(), desiredNLB)).To(Succeed()) +} + func TestNLBDeletion(t *testing.T) { var ( cs *ClusterScope diff --git a/cloud/services/loadbalancer/client.go b/cloud/services/loadbalancer/client.go index a961c107e..d0dc5e958 100644 --- a/cloud/services/loadbalancer/client.go +++ b/cloud/services/loadbalancer/client.go @@ -25,10 +25,21 @@ import ( type LoadBalancerClient interface { ListLoadBalancers(ctx context.Context, request loadbalancer.ListLoadBalancersRequest) (response loadbalancer.ListLoadBalancersResponse, err error) GetLoadBalancer(ctx context.Context, request loadbalancer.GetLoadBalancerRequest) (response loadbalancer.GetLoadBalancerResponse, err error) - CreateBackend(ctx context.Context, request loadbalancer.CreateBackendRequest) (response loadbalancer.CreateBackendResponse, err error) CreateLoadBalancer(ctx context.Context, request loadbalancer.CreateLoadBalancerRequest) (response loadbalancer.CreateLoadBalancerResponse, err error) - DeleteBackend(ctx context.Context, request loadbalancer.DeleteBackendRequest) (response loadbalancer.DeleteBackendResponse, err error) GetWorkRequest(ctx context.Context, request loadbalancer.GetWorkRequestRequest) (response loadbalancer.GetWorkRequestResponse, err error) UpdateLoadBalancer(ctx context.Context, request loadbalancer.UpdateLoadBalancerRequest) (response loadbalancer.UpdateLoadBalancerResponse, err error) DeleteLoadBalancer(ctx context.Context, request loadbalancer.DeleteLoadBalancerRequest) (response loadbalancer.DeleteLoadBalancerResponse, err error) + // BackendSet + CreateBackendSet(ctx context.Context, request loadbalancer.CreateBackendSetRequest) (response loadbalancer.CreateBackendSetResponse, err error) + DeleteBackendSet(ctx context.Context, request loadbalancer.DeleteBackendSetRequest) (response loadbalancer.DeleteBackendSetResponse, err error) + UpdateBackendSet(ctx context.Context, request loadbalancer.UpdateBackendSetRequest) (response loadbalancer.UpdateBackendSetResponse, err error) + + // Backend + CreateBackend(ctx context.Context, request loadbalancer.CreateBackendRequest) (response loadbalancer.CreateBackendResponse, err error) + DeleteBackend(ctx context.Context, request loadbalancer.DeleteBackendRequest) (response loadbalancer.DeleteBackendResponse, err error) + + // Listener + CreateListener(ctx context.Context, request loadbalancer.CreateListenerRequest) (response loadbalancer.CreateListenerResponse, err error) + DeleteListener(ctx context.Context, request loadbalancer.DeleteListenerRequest) (response loadbalancer.DeleteListenerResponse, err error) + UpdateListener(ctx context.Context, request loadbalancer.UpdateListenerRequest) (response loadbalancer.UpdateListenerResponse, err error) } diff --git a/cloud/services/loadbalancer/mock_lb/client_mock.go b/cloud/services/loadbalancer/mock_lb/client_mock.go index bf9a6f7fd..ab292769d 100644 --- a/cloud/services/loadbalancer/mock_lb/client_mock.go +++ b/cloud/services/loadbalancer/mock_lb/client_mock.go @@ -50,6 +50,21 @@ func (mr *MockLoadBalancerClientMockRecorder) CreateBackend(arg0, arg1 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackend", reflect.TypeOf((*MockLoadBalancerClient)(nil).CreateBackend), arg0, arg1) } +// CreateBackendSet mocks base method. +func (m *MockLoadBalancerClient) CreateBackendSet(arg0 context.Context, arg1 loadbalancer.CreateBackendSetRequest) (loadbalancer.CreateBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateBackendSet", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.CreateBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateBackendSet indicates an expected call of CreateBackendSet. +func (mr *MockLoadBalancerClientMockRecorder) CreateBackendSet(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackendSet", reflect.TypeOf((*MockLoadBalancerClient)(nil).CreateBackendSet), arg0, arg1) +} + // CreateLoadBalancer mocks base method. func (m *MockLoadBalancerClient) CreateLoadBalancer(arg0 context.Context, arg1 loadbalancer.CreateLoadBalancerRequest) (loadbalancer.CreateLoadBalancerResponse, error) { m.ctrl.T.Helper() @@ -65,6 +80,21 @@ func (mr *MockLoadBalancerClientMockRecorder) CreateLoadBalancer(arg0, arg1 inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLoadBalancer", reflect.TypeOf((*MockLoadBalancerClient)(nil).CreateLoadBalancer), arg0, arg1) } +// CreateListener mocks base method. +func (m *MockLoadBalancerClient) CreateListener(arg0 context.Context, arg1 loadbalancer.CreateListenerRequest) (loadbalancer.CreateListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateListener", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.CreateListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateListener indicates an expected call of CreateListener. +func (mr *MockLoadBalancerClientMockRecorder) CreateListener(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateListener", reflect.TypeOf((*MockLoadBalancerClient)(nil).CreateListener), arg0, arg1) +} + // DeleteBackend mocks base method. func (m *MockLoadBalancerClient) DeleteBackend(arg0 context.Context, arg1 loadbalancer.DeleteBackendRequest) (loadbalancer.DeleteBackendResponse, error) { m.ctrl.T.Helper() @@ -80,6 +110,36 @@ func (mr *MockLoadBalancerClientMockRecorder) DeleteBackend(arg0, arg1 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBackend", reflect.TypeOf((*MockLoadBalancerClient)(nil).DeleteBackend), arg0, arg1) } +// DeleteBackendSet mocks base method. +func (m *MockLoadBalancerClient) DeleteBackendSet(arg0 context.Context, arg1 loadbalancer.DeleteBackendSetRequest) (loadbalancer.DeleteBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBackendSet", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.DeleteBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteBackendSet indicates an expected call of DeleteBackendSet. +func (mr *MockLoadBalancerClientMockRecorder) DeleteBackendSet(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBackendSet", reflect.TypeOf((*MockLoadBalancerClient)(nil).DeleteBackendSet), arg0, arg1) +} + +// DeleteListener mocks base method. +func (m *MockLoadBalancerClient) DeleteListener(arg0 context.Context, arg1 loadbalancer.DeleteListenerRequest) (loadbalancer.DeleteListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteListener", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.DeleteListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteListener indicates an expected call of DeleteListener. +func (mr *MockLoadBalancerClientMockRecorder) DeleteListener(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteListener", reflect.TypeOf((*MockLoadBalancerClient)(nil).DeleteListener), arg0, arg1) +} + // DeleteLoadBalancer mocks base method. func (m *MockLoadBalancerClient) DeleteLoadBalancer(arg0 context.Context, arg1 loadbalancer.DeleteLoadBalancerRequest) (loadbalancer.DeleteLoadBalancerResponse, error) { m.ctrl.T.Helper() @@ -154,3 +214,33 @@ func (mr *MockLoadBalancerClientMockRecorder) UpdateLoadBalancer(arg0, arg1 inte mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLoadBalancer", reflect.TypeOf((*MockLoadBalancerClient)(nil).UpdateLoadBalancer), arg0, arg1) } + +// UpdateBackendSet mocks base method. +func (m *MockLoadBalancerClient) UpdateBackendSet(arg0 context.Context, arg1 loadbalancer.UpdateBackendSetRequest) (loadbalancer.UpdateBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateBackendSet", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.UpdateBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateBackendSet indicates an expected call of UpdateBackendSet. +func (mr *MockLoadBalancerClientMockRecorder) UpdateBackendSet(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBackendSet", reflect.TypeOf((*MockLoadBalancerClient)(nil).UpdateBackendSet), arg0, arg1) +} + +// UpdateListener mocks base method. +func (m *MockLoadBalancerClient) UpdateListener(arg0 context.Context, arg1 loadbalancer.UpdateListenerRequest) (loadbalancer.UpdateListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateListener", arg0, arg1) + ret0, _ := ret[0].(loadbalancer.UpdateListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateListener indicates an expected call of UpdateListener. +func (mr *MockLoadBalancerClientMockRecorder) UpdateListener(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateListener", reflect.TypeOf((*MockLoadBalancerClient)(nil).UpdateListener), arg0, arg1) +} diff --git a/cloud/services/networkloadbalancer/client.go b/cloud/services/networkloadbalancer/client.go index e9b386b60..e40c20f30 100644 --- a/cloud/services/networkloadbalancer/client.go +++ b/cloud/services/networkloadbalancer/client.go @@ -26,10 +26,16 @@ type NetworkLoadBalancerClient interface { ListNetworkLoadBalancers(ctx context.Context, request networkloadbalancer.ListNetworkLoadBalancersRequest) (response networkloadbalancer.ListNetworkLoadBalancersResponse, err error) GetNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.GetNetworkLoadBalancerRequest) (response networkloadbalancer.GetNetworkLoadBalancerResponse, err error) CreateBackend(ctx context.Context, request networkloadbalancer.CreateBackendRequest) (response networkloadbalancer.CreateBackendResponse, err error) + CreateBackendSet(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (response networkloadbalancer.CreateBackendSetResponse, err error) CreateNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.CreateNetworkLoadBalancerRequest) (response networkloadbalancer.CreateNetworkLoadBalancerResponse, err error) + CreateListener(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (response networkloadbalancer.CreateListenerResponse, err error) DeleteBackend(ctx context.Context, request networkloadbalancer.DeleteBackendRequest) (response networkloadbalancer.DeleteBackendResponse, err error) + DeleteBackendSet(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (response networkloadbalancer.DeleteBackendSetResponse, err error) + DeleteListener(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (response networkloadbalancer.DeleteListenerResponse, err error) GetWorkRequest(ctx context.Context, request networkloadbalancer.GetWorkRequestRequest) (response networkloadbalancer.GetWorkRequestResponse, err error) ListWorkRequestErrors(ctx context.Context, request networkloadbalancer.ListWorkRequestErrorsRequest) (response networkloadbalancer.ListWorkRequestErrorsResponse, err error) + UpdateBackendSet(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (response networkloadbalancer.UpdateBackendSetResponse, err error) + UpdateListener(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (response networkloadbalancer.UpdateListenerResponse, err error) UpdateNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.UpdateNetworkLoadBalancerRequest) (response networkloadbalancer.UpdateNetworkLoadBalancerResponse, err error) DeleteNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.DeleteNetworkLoadBalancerRequest) (response networkloadbalancer.DeleteNetworkLoadBalancerResponse, err error) } diff --git a/cloud/services/networkloadbalancer/mock_nlb/client_mock.go b/cloud/services/networkloadbalancer/mock_nlb/client_mock.go index 888bc29fc..747c6a545 100644 --- a/cloud/services/networkloadbalancer/mock_nlb/client_mock.go +++ b/cloud/services/networkloadbalancer/mock_nlb/client_mock.go @@ -50,6 +50,21 @@ func (mr *MockNetworkLoadBalancerClientMockRecorder) CreateBackend(ctx, request return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackend", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).CreateBackend), ctx, request) } +// CreateBackendSet mocks base method. +func (m *MockNetworkLoadBalancerClient) CreateBackendSet(ctx context.Context, request networkloadbalancer.CreateBackendSetRequest) (networkloadbalancer.CreateBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateBackendSet", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.CreateBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateBackendSet indicates an expected call of CreateBackendSet. +func (mr *MockNetworkLoadBalancerClientMockRecorder) CreateBackendSet(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackendSet", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).CreateBackendSet), ctx, request) +} + // CreateNetworkLoadBalancer mocks base method. func (m *MockNetworkLoadBalancerClient) CreateNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.CreateNetworkLoadBalancerRequest) (networkloadbalancer.CreateNetworkLoadBalancerResponse, error) { m.ctrl.T.Helper() @@ -65,6 +80,21 @@ func (mr *MockNetworkLoadBalancerClientMockRecorder) CreateNetworkLoadBalancer(c return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNetworkLoadBalancer", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).CreateNetworkLoadBalancer), ctx, request) } +// CreateListener mocks base method. +func (m *MockNetworkLoadBalancerClient) CreateListener(ctx context.Context, request networkloadbalancer.CreateListenerRequest) (networkloadbalancer.CreateListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateListener", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.CreateListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateListener indicates an expected call of CreateListener. +func (mr *MockNetworkLoadBalancerClientMockRecorder) CreateListener(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateListener", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).CreateListener), ctx, request) +} + // DeleteBackend mocks base method. func (m *MockNetworkLoadBalancerClient) DeleteBackend(ctx context.Context, request networkloadbalancer.DeleteBackendRequest) (networkloadbalancer.DeleteBackendResponse, error) { m.ctrl.T.Helper() @@ -80,6 +110,36 @@ func (mr *MockNetworkLoadBalancerClientMockRecorder) DeleteBackend(ctx, request return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBackend", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).DeleteBackend), ctx, request) } +// DeleteBackendSet mocks base method. +func (m *MockNetworkLoadBalancerClient) DeleteBackendSet(ctx context.Context, request networkloadbalancer.DeleteBackendSetRequest) (networkloadbalancer.DeleteBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBackendSet", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.DeleteBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteBackendSet indicates an expected call of DeleteBackendSet. +func (mr *MockNetworkLoadBalancerClientMockRecorder) DeleteBackendSet(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBackendSet", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).DeleteBackendSet), ctx, request) +} + +// DeleteListener mocks base method. +func (m *MockNetworkLoadBalancerClient) DeleteListener(ctx context.Context, request networkloadbalancer.DeleteListenerRequest) (networkloadbalancer.DeleteListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteListener", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.DeleteListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteListener indicates an expected call of DeleteListener. +func (mr *MockNetworkLoadBalancerClientMockRecorder) DeleteListener(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteListener", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).DeleteListener), ctx, request) +} + // DeleteNetworkLoadBalancer mocks base method. func (m *MockNetworkLoadBalancerClient) DeleteNetworkLoadBalancer(ctx context.Context, request networkloadbalancer.DeleteNetworkLoadBalancerRequest) (networkloadbalancer.DeleteNetworkLoadBalancerResponse, error) { m.ctrl.T.Helper() @@ -169,3 +229,33 @@ func (mr *MockNetworkLoadBalancerClientMockRecorder) UpdateNetworkLoadBalancer(c mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateNetworkLoadBalancer", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).UpdateNetworkLoadBalancer), ctx, request) } + +// UpdateBackendSet mocks base method. +func (m *MockNetworkLoadBalancerClient) UpdateBackendSet(ctx context.Context, request networkloadbalancer.UpdateBackendSetRequest) (networkloadbalancer.UpdateBackendSetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateBackendSet", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.UpdateBackendSetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateBackendSet indicates an expected call of UpdateBackendSet. +func (mr *MockNetworkLoadBalancerClientMockRecorder) UpdateBackendSet(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBackendSet", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).UpdateBackendSet), ctx, request) +} + +// UpdateListener mocks base method. +func (m *MockNetworkLoadBalancerClient) UpdateListener(ctx context.Context, request networkloadbalancer.UpdateListenerRequest) (networkloadbalancer.UpdateListenerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateListener", ctx, request) + ret0, _ := ret[0].(networkloadbalancer.UpdateListenerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateListener indicates an expected call of UpdateListener. +func (mr *MockNetworkLoadBalancerClientMockRecorder) UpdateListener(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateListener", reflect.TypeOf((*MockNetworkLoadBalancerClient)(nil).UpdateListener), ctx, request) +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml index 072611e28..11e3b24c6 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml @@ -133,8 +133,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections will @@ -165,6 +166,66 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend set. + properties: + backendSetDetails: + description: BackendSetDetails specifies the backend + set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer + name: + description: Name is the API server backend set + identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID @@ -1294,8 +1355,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections will @@ -1326,6 +1388,66 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend set. + properties: + backendSetDetails: + description: BackendSetDetails specifies the backend + set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer + name: + description: Name is the API server backend set + identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml index 9be0185ea..4d3d0c088 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclustertemplates.yaml @@ -146,8 +146,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections @@ -180,6 +181,67 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend + set. + properties: + backendSetDetails: + description: BackendSetDetails specifies + the backend set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer + name: + description: Name is the API server backend + set identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID @@ -1252,8 +1314,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections @@ -1286,6 +1349,67 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend + set. + properties: + backendSetDetails: + description: BackendSetDetails specifies + the backend set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer + name: + description: Name is the API server backend + set identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml index eb15a2175..6435364ad 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclusters.yaml @@ -136,8 +136,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections will @@ -168,6 +169,66 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend set. + properties: + backendSetDetails: + description: BackendSetDetails specifies the backend + set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer + name: + description: Name is the API server backend set + identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID @@ -1300,8 +1361,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections will @@ -1332,6 +1394,66 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend set. + properties: + backendSetDetails: + description: BackendSetDetails specifies the backend + set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative healthy + backend as soon as current backend becomes + unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer + name: + description: Name is the API server backend set + identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml index 1ff3babbe..737aec6d2 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedclustertemplates.yaml @@ -151,8 +151,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections @@ -185,6 +186,67 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend + set. + properties: + backendSetDetails: + description: BackendSetDetails specifies + the backend set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer + name: + description: Name is the API server backend + set identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID @@ -1262,8 +1324,9 @@ spec: description: The NLB Spec properties: backendSetDetails: - description: BackendSetDetails specifies the configuration - of a network load balancer backend set. + description: |- + BackendSetDetails specifies the configuration of a network load balancer backend set. + Deprecated: use `backendSets` instead. properties: healthChecker: description: If enabled existing connections @@ -1296,6 +1359,67 @@ spec: The value is false by default. type: boolean type: object + backendSets: + description: |- + BackendSets specifies the canonical list of API server backend sets. + When set, this field takes precedence over the legacy `backendSetDetails` field. + items: + description: NLBBackendSet specifies the configuration + for a named network load balancer backend + set. + properties: + backendSetDetails: + description: BackendSetDetails specifies + the backend set behavior. + properties: + healthChecker: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + properties: + urlPath: + description: |- + The path against which to run the health check. + Example: `/healthcheck` + Default value is `/healthz` + type: string + type: object + isFailOpen: + description: |- + If enabled, the network load balancer will continue to distribute traffic in the configured distribution in the event all backends are unhealthy. + The value is false by default. + type: boolean + isInstantFailoverEnabled: + description: If enabled existing connections + will be forwarded to an alternative + healthy backend as soon as current + backend becomes unhealthy. + type: boolean + isPreserveSource: + description: |- + If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends. + Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled. + The value is false by default. + type: boolean + type: object + listenerPort: + description: |- + ListenerPort is the load balancer listener port routed to this backend set. + When omitted, the cluster API server port is used. + format: int32 + type: integer + name: + description: Name is the API server backend + set identifier. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map reservedIpIds: description: |- A list of a reserved Ip OCID diff --git a/docs/src/networking/custom-networking.md b/docs/src/networking/custom-networking.md index 83cceb742..303ddbb9a 100644 --- a/docs/src/networking/custom-networking.md +++ b/docs/src/networking/custom-networking.md @@ -300,6 +300,82 @@ spec: loadBalancerType: "lb" ``` +## Configure multiple API-server backend sets and listeners + +CAPOCI supports multiple API-server backend sets through `networkSpec.apiServerLoadBalancer.nlbSpec.backendSets`. +Each backend set can optionally define a dedicated `listenerPort`. + +Field precedence and compatibility rules: + +1. `backendSets` is the canonical field. +2. Legacy `backendSetDetails` remains supported for backward compatibility. +3. If both are set, `backendSets` takes precedence and validation rejects mixed usage. +4. If only legacy `backendSetDetails` is set, CAPOCI preserves single-backend-set behavior. + +Example (NLB default) with multiple listeners: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + name: "${CLUSTER_NAME}" +spec: + compartmentId: "${OCI_COMPARTMENT_ID}" + networkSpec: + apiServerLoadBalancer: + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: rke2-registration + listenerPort: 9345 + backendSetDetails: + healthChecker: + urlPath: /v1-rke2/readyz +``` + +Equivalent configuration for OCI LBaaS is supported by setting `loadBalancerType: "lb"`: + +> Note: `nlbSpec` is currently the shared API-server load balancer configuration field for both OCI NLB and OCI LBaaS. The field name is historical; LB multi-backend-set configuration is also defined under `nlbSpec`. + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + name: "${CLUSTER_NAME}" +spec: + compartmentId: "${OCI_COMPARTMENT_ID}" + networkSpec: + apiServerLoadBalancer: + loadBalancerType: "lb" + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + - name: rke2-registration + listenerPort: 9345 +``` + +### Migration from legacy `backendSetDetails` + +Use this sequence: + +1. Keep your existing legacy `backendSetDetails` (no behavior change). +2. Replace it with canonical `backendSets` and include the default set (`apiserver-lb-backendset`) first. +3. Add additional backend sets with unique names and optional `listenerPort`. +4. Apply manifests and wait for reconcile to create/update listeners and backend sets. +5. During rollout, old/new backend sets can coexist; machines are reconciled into active sets. +6. After transition, remove no-longer-needed backend sets from `backendSets`. CAPOCI garbage-collects stale listeners and stale backend sets when safe (empty/unreferenced). + +### Rollout semantics and caveats + +- The first canonical backend set remains the primary API-server listener target (`apiserver-lb-listener`). +- Additional backend sets create additional deterministic listeners for management endpoints. +- `listenerPort` values must be unique and within `1-65535`. +- Backend set names must be unique, and match provider constraints (alphanumeric, `-`, `_`). +- CAPOCI does not require immediate manual migration for old objects; legacy manifests continue to decode. + ## Example spec to use custom role CAPOCI can be used to create Subnet/NSG in the VCN for custom workloads such as private load balancers, diff --git a/templates/cluster-template-multiple-backends-lb.yaml b/templates/cluster-template-multiple-backends-lb.yaml new file mode 100644 index 000000000..91e9d8883 --- /dev/null +++ b/templates/cluster-template-multiple-backends-lb.yaml @@ -0,0 +1,154 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: OCICluster + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + compartmentId: "${OCI_COMPARTMENT_ID}" + networkSpec: + apiServerLoadBalancer: + loadBalancerType: "lb" + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: apiserver-lb-backendset-2 + listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} + backendSetDetails: + healthChecker: + urlPath: /readyz +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + version: "${KUBERNETES_VERSION}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: OCIMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: ${KUBERNETES_VERSION} + apiServer: + certSANs: [localhost, 127.0.0.1] + dns: {} + etcd: {} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +kind: OCIMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + imageId: "${OCI_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_CONTROL_PLANE_MACHINE_TYPE=VM.Standard.E5.Flex}" + shapeConfig: + ocpus: "${OCI_CONTROL_PLANE_MACHINE_TYPE_OCPUS=1}" + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_CONTROL_PLANE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCIMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + imageId: "${OCI_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_NODE_MACHINE_TYPE=VM.Standard.E5.Flex}" + shapeConfig: + ocpus: "${OCI_NODE_MACHINE_TYPE_OCPUS=1}" + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${NODE_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: OCIMachineTemplate diff --git a/templates/cluster-template-multiple-backends.yaml b/templates/cluster-template-multiple-backends.yaml new file mode 100644 index 000000000..b74deab90 --- /dev/null +++ b/templates/cluster-template-multiple-backends.yaml @@ -0,0 +1,153 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: OCICluster + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + compartmentId: "${OCI_COMPARTMENT_ID}" + networkSpec: + apiServerLoadBalancer: + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: apiserver-lb-backendset-2 + listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} + backendSetDetails: + healthChecker: + urlPath: /readyz +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + version: "${KUBERNETES_VERSION}" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: OCIMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: ${KUBERNETES_VERSION} + apiServer: + certSANs: [localhost, 127.0.0.1] + dns: {} + etcd: {} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +kind: OCIMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + imageId: "${OCI_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_CONTROL_PLANE_MACHINE_TYPE=VM.Standard.E5.Flex}" + shapeConfig: + ocpus: "${OCI_CONTROL_PLANE_MACHINE_TYPE_OCPUS=1}" + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_CONTROL_PLANE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCIMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + imageId: "${OCI_IMAGE_ID}" + compartmentId: "${OCI_COMPARTMENT_ID}" + shape: "${OCI_NODE_MACHINE_TYPE=VM.Standard.E5.Flex}" + shapeConfig: + ocpus: "${OCI_NODE_MACHINE_TYPE_OCPUS=1}" + metadata: + ssh_authorized_keys: "${OCI_SSH_KEY}" + isPvEncryptionInTransitEnabled: ${OCI_NODE_PV_TRANSIT_ENCRYPTION=true} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${NODE_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: OCIMachineTemplate diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index 8074389bc..fc5fc1b8f 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -31,9 +31,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" infrastructurev1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1" + infrastructurev1beta2 "github.com/oracle/cluster-api-provider-oci/api/v1beta2" "github.com/oracle/cluster-api-provider-oci/cloud/scope" "github.com/oracle/oci-go-sdk/v65/common" "github.com/oracle/oci-go-sdk/v65/core" + "github.com/oracle/oci-go-sdk/v65/loadbalancer" "github.com/oracle/oci-go-sdk/v65/networkloadbalancer" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -156,6 +158,52 @@ var _ = Describe("Workload cluster creation", func() { validateFailureDomainSpread(namespace.Name, clusterName) }) + It("Multiple API server backend sets on NLB [PRBlocking]", func() { + clusterName = getClusterName(clusterNamePrefix, "multiple-backends-nlb") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "multiple-backends-nlb", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.MustGetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64(1), + WorkerMachineCount: pointer.Int64(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + verifyAdditionalAPIServerBackendSetResources(ctx, specName, namespace.Name, clusterName) + }) + + It("Multiple API server backend sets on LB [DailyTests]", func() { + clusterName = getClusterName(clusterNamePrefix, "multiple-backends-lb") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "multiple-backends-lb", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.MustGetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64(1), + WorkerMachineCount: pointer.Int64(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + verifyAdditionalAPIServerBackendSetResources(ctx, specName, namespace.Name, clusterName) + }) + It("Antrea as CNI - With 1 control-plane nodes and 1 worker nodes [DailyTests]", func() { clusterName = getClusterName(clusterNamePrefix, "antrea") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ @@ -874,8 +922,128 @@ func validateVnicNSG(ctx context.Context, clusterName string, nameSpace string, Expect(vnic.NsgIds[0]).To(Equal(*nsgId)) } } - Expect(exists).To(Equal(true)) } + Expect(exists).To(Equal(true)) +} + +func verifyAdditionalAPIServerBackendSetResources(ctx context.Context, specName, namespaceName, clusterName string) { + Eventually(func() error { + k8sClient := bootstrapClusterProxy.GetClient() + ociCluster := &infrastructurev1beta2.OCICluster{} + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespaceName, Name: clusterName}, ociCluster); err != nil { + return err + } + lbID := ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerId + if lbID == nil || *lbID == "" { + return fmt.Errorf("api server load balancer ID not yet assigned") + } + + expectedBackendSets := ociCluster.Spec.NetworkSpec.APIServerLB.CanonicalAPIServerBackendSets() + if len(expectedBackendSets) < 2 { + return fmt.Errorf("expected multiple API server backend sets, got %d", len(expectedBackendSets)) + } + + defaultPort := ociCluster.Spec.ControlPlaneEndpoint.Port + if defaultPort == 0 { + defaultPort = 6443 + } + + switch ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerType { + case infrastructurev1beta2.LoadBalancerTypeLB: + return verifyAPIServerLBBackendSets(ctx, *lbID, defaultPort, expectedBackendSets) + case "", infrastructurev1beta2.LoadBalancerTypeNLB: + return verifyAPIServerNLBBackendSets(ctx, *lbID, defaultPort, expectedBackendSets) + default: + return fmt.Errorf("unsupported api server load balancer type %q", ociCluster.Spec.NetworkSpec.APIServerLB.LoadBalancerType) + } + }, e2eConfig.GetIntervals(specName, "wait-control-plane")...).Should(Succeed()) +} + +func verifyAPIServerNLBBackendSets(ctx context.Context, lbID string, defaultPort int32, expectedBackendSets []infrastructurev1beta2.NLBBackendSet) error { + resp, err := lbClient.GetNetworkLoadBalancer(ctx, networkloadbalancer.GetNetworkLoadBalancerRequest{ + NetworkLoadBalancerId: common.String(lbID), + }) + if err != nil { + return err + } + + if len(resp.NetworkLoadBalancer.BackendSets) != len(expectedBackendSets) { + return fmt.Errorf("expected %d nlb backend sets, got %d", len(expectedBackendSets), len(resp.NetworkLoadBalancer.BackendSets)) + } + if len(resp.NetworkLoadBalancer.Listeners) != len(expectedBackendSets) { + return fmt.Errorf("expected %d nlb listeners, got %d", len(expectedBackendSets), len(resp.NetworkLoadBalancer.Listeners)) + } + + for _, backendSet := range expectedBackendSets { + if _, ok := resp.NetworkLoadBalancer.BackendSets[backendSet.Name]; !ok { + return fmt.Errorf("nlb backend set %q not found", backendSet.Name) + } + if !hasExpectedNLBListener(resp.NetworkLoadBalancer.Listeners, backendSet.Name, expectedAPIServerListenerPort(defaultPort, backendSet)) { + return fmt.Errorf("nlb listener for backend set %q on port %d not found", backendSet.Name, expectedAPIServerListenerPort(defaultPort, backendSet)) + } + } + + return nil +} + +func verifyAPIServerLBBackendSets(ctx context.Context, lbID string, defaultPort int32, expectedBackendSets []infrastructurev1beta2.NLBBackendSet) error { + resp, err := lbaasClient.GetLoadBalancer(ctx, loadbalancer.GetLoadBalancerRequest{ + LoadBalancerId: common.String(lbID), + }) + if err != nil { + return err + } + + if len(resp.LoadBalancer.BackendSets) != len(expectedBackendSets) { + return fmt.Errorf("expected %d lb backend sets, got %d", len(expectedBackendSets), len(resp.LoadBalancer.BackendSets)) + } + if len(resp.LoadBalancer.Listeners) != len(expectedBackendSets) { + return fmt.Errorf("expected %d lb listeners, got %d", len(expectedBackendSets), len(resp.LoadBalancer.Listeners)) + } + + for _, backendSet := range expectedBackendSets { + if _, ok := resp.LoadBalancer.BackendSets[backendSet.Name]; !ok { + return fmt.Errorf("lb backend set %q not found", backendSet.Name) + } + if !hasExpectedLBListener(resp.LoadBalancer.Listeners, backendSet.Name, expectedAPIServerListenerPort(defaultPort, backendSet)) { + return fmt.Errorf("lb listener for backend set %q on port %d not found", backendSet.Name, expectedAPIServerListenerPort(defaultPort, backendSet)) + } + } + + return nil +} + +func hasExpectedNLBListener(listeners map[string]networkloadbalancer.Listener, backendSetName string, port int32) bool { + for _, listener := range listeners { + if listener.DefaultBackendSetName == nil || *listener.DefaultBackendSetName != backendSetName { + continue + } + if listener.Port == nil || int32(*listener.Port) != port { + continue + } + return true + } + return false +} + +func hasExpectedLBListener(listeners map[string]loadbalancer.Listener, backendSetName string, port int32) bool { + for _, listener := range listeners { + if listener.DefaultBackendSetName == nil || *listener.DefaultBackendSetName != backendSetName { + continue + } + if listener.Port == nil || int32(*listener.Port) != port { + continue + } + return true + } + return false +} + +func expectedAPIServerListenerPort(defaultPort int32, backendSet infrastructurev1beta2.NLBBackendSet) int32 { + if backendSet.ListenerPort != nil { + return *backendSet.ListenerPort + } + return defaultPort } func validateCustomNetworkingSeclist(ctx context.Context, ociCluster infrastructurev1beta1.OCICluster, clusterName string) error { diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 7772f86ba..fff2eefbb 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -63,6 +63,8 @@ providers: - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-custom-networking-seclist.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-custom-networking-nsg.yaml" - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-multiple-node-nsg.yaml" + - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb.yaml" + - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb.yaml" - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template.yaml" - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-cluster-class.yaml" - sourcePath: "../data/infrastructure-oci/v1beta2/cluster-template-cluster-class/clusterclass-test-cluster-class.yaml" diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml new file mode 100644 index 000000000..099de5385 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/cluster.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + name: "${CLUSTER_NAME}" +spec: + networkSpec: + apiServerLoadBalancer: + loadBalancerType: "lb" + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: apiserver-lb-backendset-2 + listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} + backendSetDetails: + healthChecker: + urlPath: /readyz diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/kustomization.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/kustomization.yaml new file mode 100644 index 000000000..a95684fe6 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-lb/kustomization.yaml @@ -0,0 +1,7 @@ +bases: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../../bases/crs.yaml + - ../../bases/ccm.yaml +patchesStrategicMerge: + - ./cluster.yaml diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml new file mode 100644 index 000000000..59cdf2de9 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/cluster.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: OCICluster +metadata: + name: "${CLUSTER_NAME}" +spec: + networkSpec: + apiServerLoadBalancer: + nlbSpec: + backendSets: + - name: apiserver-lb-backendset + backendSetDetails: + healthChecker: + urlPath: /healthz + - name: apiserver-lb-backendset-2 + listenerPort: ${API_SERVER_SECONDARY_LISTENER_PORT:=9345} + backendSetDetails: + healthChecker: + urlPath: /readyz diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/kustomization.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/kustomization.yaml new file mode 100644 index 000000000..a95684fe6 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-multiple-backends-nlb/kustomization.yaml @@ -0,0 +1,7 @@ +bases: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../../bases/crs.yaml + - ../../bases/ccm.yaml +patchesStrategicMerge: + - ./cluster.yaml diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 59fce299c..1bf4f990c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -41,6 +41,7 @@ import ( "github.com/oracle/cluster-api-provider-oci/cloud/scope" "github.com/oracle/cluster-api-provider-oci/cloud/services/compute" "github.com/oracle/cluster-api-provider-oci/cloud/services/containerengine" + lb "github.com/oracle/cluster-api-provider-oci/cloud/services/loadbalancer" nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer" "github.com/oracle/cluster-api-provider-oci/cloud/services/vcn" infrav1exp "github.com/oracle/cluster-api-provider-oci/exp/api/v1beta1" @@ -107,7 +108,8 @@ var ( vcnClient vcn.Client - lbClient nlb.NetworkLoadBalancerClient + lbClient nlb.NetworkLoadBalancerClient + lbaasClient lb.LoadBalancerClient okeClient containerengine.Client @@ -246,9 +248,11 @@ var _ = SynchronizedBeforeSuite(func() []byte { identityClient := ociClients.IdentityClient vcnClient = ociClients.VCNClient lbClient = ociClients.NetworkLoadBalancerClient + lbaasClient = ociClients.LoadBalancerClient okeClient = ociClients.ContainerEngineClient Expect(identityClient).NotTo(BeNil()) Expect(lbClient).NotTo(BeNil()) + Expect(lbaasClient).NotTo(BeNil()) req := identity.ListAvailabilityDomainsRequest{CompartmentId: common.String(os.Getenv("OCI_COMPARTMENT_ID"))} resp, err := identityClient.ListAvailabilityDomains(context.Background(), req)