[WIP] feat: multipl backend sets lb

Makefile

@@ -287,6 +287,8 @@ generate-e2e-templates: $(KUSTOMIZE)
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-custom-networking-seclist --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-custom-networking-seclist.yaml
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-custom-networking-nsg --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-custom-networking-nsg.yaml
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-node-nsg --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-node-nsg.yaml
+	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-nlb --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-nlb.yaml
+	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-lb --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-multiple-backends-lb.yaml
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-cluster-class --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-cluster-class.yaml
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-local-vcn-peering --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-local-vcn-peering.yaml
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta2/cluster-template-remote-vcn-peering --load-restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta2/cluster-template-remote-vcn-peering.yaml

api/internal/apiserverlb/backendsets.go

@@ -0,0 +1,153 @@
+package apiserverlb
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/oracle/cluster-api-provider-oci/cloud/ociutil"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+)
+
+const BackendSetNameRegex = "^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$"
+
+type BackendSet struct {
+	Name         string
+	ListenerPort *int32
+}
+
+func CanonicalBackendSets[T any](configured []T, synthesize func() T) []T {
+	if len(configured) > 0 {
+		return append([]T(nil), configured...)
+	}
+	return []T{synthesize()}
+}
+
+func ValidateBackendSets(backendSets []BackendSet, legacyConfigured bool, apiServerPort *int32, supportedSecondaryListenerPort int32, fldPath *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+
+	if len(backendSets) > 0 && legacyConfigured {
+		allErrs = append(allErrs, field.Forbidden(
+			fldPath.Child("backendSetDetails"),
+			"cannot be set when backendSets is configured; move legacy backendSetDetails into backendSets",
+		))
+	}
+
+	nameRegex := regexp.MustCompile(BackendSetNameRegex)
+	seen := map[string]int{}
+	for i, backendSet := range backendSets {
+		namePath := fldPath.Child("backendSets").Index(i).Child("name")
+		name := strings.TrimSpace(backendSet.Name)
+		if name == "" {
+			allErrs = append(allErrs, field.Required(namePath, "must not be empty"))
+			continue
+		}
+		if !nameRegex.MatchString(name) {
+			allErrs = append(allErrs, field.Invalid(
+				namePath,
+				backendSet.Name,
+				"must match ^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$ (1-32 chars; alphanumeric, '-', '_')",
+			))
+		}
+		if firstIdx, ok := seen[name]; ok {
+			allErrs = append(allErrs, field.Invalid(
+				namePath,
+				backendSet.Name,
+				fmt.Sprintf("duplicate backend set name, already used at backendSets[%d].name", firstIdx),
+			))
+			continue
+		}
+		seen[name] = i
+	}
+
+	usedPorts := map[int32]int{}
+	omittedListenerPortIndex := -1
+	for i, backendSet := range backendSets {
+		portPath := fldPath.Child("backendSets").Index(i).Child("listenerPort")
+		if backendSet.ListenerPort != nil {
+			port := *backendSet.ListenerPort
+			if port < 1 || port > 65535 {
+				allErrs = append(allErrs, field.Invalid(portPath, port, "must be between 1 and 65535"))
+				continue
+			}
+			if !IsSupportedAPIServerListenerPort(port, apiServerPort, supportedSecondaryListenerPort) {
+				allErrs = append(allErrs, field.Invalid(
+					portPath,
+					port,
+					fmt.Sprintf("must be one of %s to avoid exposing arbitrary control-plane ports", SupportedAPIServerListenerPortsString(apiServerPort, supportedSecondaryListenerPort)),
+				))
+				continue
+			}
+		}
+
+		port, known := EffectiveAPIServerListenerPort(backendSet.ListenerPort, apiServerPort)
+		if !known {
+			if omittedListenerPortIndex >= 0 {
+				allErrs = append(allErrs, field.Invalid(
+					portPath,
+					backendSet.ListenerPort,
+					fmt.Sprintf("multiple backend sets omit listenerPort, already omitted at backendSets[%d].listenerPort", omittedListenerPortIndex),
+				))
+				continue
+			}
+			omittedListenerPortIndex = i
+			continue
+		}
+
+		if firstIdx, ok := usedPorts[port]; ok {
+			allErrs = append(allErrs, field.Invalid(
+				portPath,
+				backendSet.ListenerPort,
+				fmt.Sprintf("duplicate effective listenerPort %d, already used at backendSets[%d].listenerPort", port, firstIdx),
+			))
+			continue
+		}
+		usedPorts[port] = i
+	}
+
+	return allErrs
+}
+
+func EffectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) {
+	if listenerPort != nil {
+		return *listenerPort, true
+	}
+	if apiServerPort == nil {
+		return 0, false
+	}
+	return *apiServerPort, true
+}
+
+func IsSupportedAPIServerListenerPort(port int32, apiServerPort *int32, supportedSecondaryListenerPort int32) bool {
+	for _, allowedPort := range SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort) {
+		if port == allowedPort {
+			return true
+		}
+	}
+	return false
+}
+
+func SupportedAPIServerListenerPorts(apiServerPort *int32, supportedSecondaryListenerPort int32) []int32 {
+	ports := []int32{ociutil.DefaultAPIServerPort}
+	if apiServerPort != nil && *apiServerPort != ociutil.DefaultAPIServerPort {
+		ports = append(ports, *apiServerPort)
+	}
+	if supportedSecondaryListenerPort != ociutil.DefaultAPIServerPort {
+		ports = append(ports, supportedSecondaryListenerPort)
+	}
+	return ports
+}
+
+func SupportedAPIServerListenerPortsString(apiServerPort *int32, supportedSecondaryListenerPort int32) string {
+	values := SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort)
+	parts := make([]string, 0, len(values))
+	seen := map[int32]struct{}{}
+	for _, value := range values {
+		if _, ok := seen[value]; ok {
+			continue
+		}
+		seen[value] = struct{}{}
+		parts = append(parts, fmt.Sprintf("%d", value))
+	}
+	return strings.Join(parts, ", ")
+}

api/v1beta1/backendsets_shared.go

@@ -0,0 +1,42 @@
+package v1beta1
+
+import (
+	apiserverlb "github.com/oracle/cluster-api-provider-oci/api/internal/apiserverlb"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+)
+
+func canonicalAPIServerBackendSets(configured []NLBBackendSet, legacy BackendSetDetails) []NLBBackendSet {
+	return apiserverlb.CanonicalBackendSets(configured, func() NLBBackendSet {
+		return NLBBackendSet{
+			Name:              APIServerLBBackendSetName,
+			BackendSetDetails: legacy,
+		}
+	})
+}
+
+func validateSharedAPIServerLBBackendSets(spec NLBSpec, apiServerPort *int32, fldPath *field.Path) field.ErrorList {
+	backendSets := make([]apiserverlb.BackendSet, 0, len(spec.BackendSets))
+	for _, backendSet := range spec.BackendSets {
+		backendSets = append(backendSets, apiserverlb.BackendSet{
+			Name:         backendSet.Name,
+			ListenerPort: backendSet.ListenerPort,
+		})
+	}
+	return apiserverlb.ValidateBackendSets(backendSets, spec.BackendSetDetails != (BackendSetDetails{}), apiServerPort, supportedSecondaryListenerPort, fldPath)
+}
+
+func effectiveAPIServerListenerPort(listenerPort *int32, apiServerPort *int32) (int32, bool) {
+	return apiserverlb.EffectiveAPIServerListenerPort(listenerPort, apiServerPort)
+}
+
+func isSupportedAPIServerListenerPort(port int32, apiServerPort *int32) bool {
+	return apiserverlb.IsSupportedAPIServerListenerPort(port, apiServerPort, supportedSecondaryListenerPort)
+}
+
+func supportedAPIServerListenerPorts(apiServerPort *int32) []int32 {
+	return apiserverlb.SupportedAPIServerListenerPorts(apiServerPort, supportedSecondaryListenerPort)
+}
+
+func supportedAPIServerListenerPortsString(apiServerPort *int32) string {
+	return apiserverlb.SupportedAPIServerListenerPortsString(apiServerPort, supportedSecondaryListenerPort)
+}

api/v1beta1/ocicluster_conversion_test.go

@@ -25,6 +25,83 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/conversion"
 )
 
+func TestOCICluster_BackendSetsRoundTrip(t *testing.T) {
+	t.Run("v1beta1 legacy -> v1beta2 -> v1beta1 keeps legacy shape", func(t *testing.T) {
+		src := &OCICluster{
+			Spec: OCIClusterSpec{
+				NetworkSpec: NetworkSpec{
+					APIServerLB: LoadBalancer{
+						NLBSpec: NLBSpec{
+							BackendSetDetails: BackendSetDetails{
+								IsFailOpen: boolPtr(true),
+							},
+						},
+					},
+				},
+			},
+		}
+
+		hub := &v1beta2.OCICluster{}
+		if err := src.ConvertTo(hub); err != nil {
+			t.Fatalf("convert to hub failed: %v", err)
+		}
+		if len(hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) != 0 {
+			t.Fatalf("expected no canonical backendSets to be synthesized during conversion, got %#v", hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets)
+		}
+		if hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen == nil || !*hub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen {
+			t.Fatalf("expected legacy backendSetDetails to be preserved on hub")
+		}
+
+		restored := &OCICluster{}
+		if err := restored.ConvertFrom(hub); err != nil {
+			t.Fatalf("convert from hub failed: %v", err)
+		}
+		if restored.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen == nil || !*restored.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails.IsFailOpen {
+			t.Fatalf("expected legacy backendSetDetails after roundtrip, got %#v", restored.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSetDetails)
+		}
+	})
+
+	t.Run("v1beta2 canonical -> v1beta1 -> v1beta2 keeps canonical shape", func(t *testing.T) {
+		srcHub := &v1beta2.OCICluster{
+			Spec: v1beta2.OCIClusterSpec{
+				NetworkSpec: v1beta2.NetworkSpec{
+					APIServerLB: v1beta2.LoadBalancer{
+						NLBSpec: v1beta2.NLBSpec{
+							BackendSets: []v1beta2.NLBBackendSet{
+								{
+									Name: "new-set",
+									BackendSetDetails: v1beta2.BackendSetDetails{
+										IsFailOpen: boolPtr(true),
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		spoke := &OCICluster{}
+		if err := spoke.ConvertFrom(srcHub); err != nil {
+			t.Fatalf("convert from hub failed: %v", err)
+		}
+		if len(spoke.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) != 1 || spoke.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].Name != "new-set" {
+			t.Fatalf("expected canonical backendSets after down-conversion, got %#v", spoke.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets)
+		}
+
+		restoredHub := &v1beta2.OCICluster{}
+		if err := spoke.ConvertTo(restoredHub); err != nil {
+			t.Fatalf("convert back to hub failed: %v", err)
+		}
+		if len(restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets) != 1 || restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].Name != "new-set" {
+			t.Fatalf("expected canonical backendSets after hub roundtrip, got %#v", restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets)
+		}
+		if restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].BackendSetDetails.IsFailOpen == nil || !*restoredHub.Spec.NetworkSpec.APIServerLB.NLBSpec.BackendSets[0].BackendSetDetails.IsFailOpen {
+			t.Fatalf("expected backendSetDetails to remain intact after roundtrip")
+		}
+	})
+}
+
 func TestOCICluster_ConvertTo(t *testing.T) {
 	type fields struct {
 		TypeMeta   metav1.TypeMeta

api/v1beta1/types.go

@@ -982,14 +982,43 @@ type LoadBalancer struct {
 	// +optional
 	LoadBalancerId *string `json:"loadBalancerId,omitempty"`
 
-	// The NLB Spec
+	// Shared API server load balancer settings used for both `loadBalancerType: nlb` and `loadBalancerType: lb`.
+	// The field name is historical; it is not limited to the NLB implementation.
 	// +optional
 	NLBSpec NLBSpec `json:"nlbSpec,omitempty"`
 }
 
-// NLBSpec specifies the NLB spec.
+// NewAPIServerLoadBalancer returns a LoadBalancer configured with the shared API server settings.
+// The underlying `nlbSpec` field name is historical and retained for API compatibility.
+func NewAPIServerLoadBalancer(name string, spec NLBSpec) LoadBalancer {
+	return LoadBalancer{
+		Name:    name,
+		NLBSpec: spec,
+	}
+}
+
+// APIServerLoadBalancerSpec returns the shared API server load balancer settings.
+func (in *LoadBalancer) APIServerLoadBalancerSpec() *NLBSpec {
+	return &in.NLBSpec
+}
+
+// CanonicalAPIServerBackendSets returns the canonical API server backend set list for this load balancer.
+func (in *LoadBalancer) CanonicalAPIServerBackendSets() []NLBBackendSet {
+	return in.APIServerLoadBalancerSpec().CanonicalBackendSets()
+}
+
+// NLBSpec specifies the shared API server load balancer settings.
+// The type name is historical and applies to both `loadBalancerType: nlb` and `loadBalancerType: lb`.
 type NLBSpec struct {
+	// BackendSets specifies the canonical list of API server backend sets.
+	// When set, this field takes precedence over the legacy `backendSetDetails` field.
+	// +optional
+	// +listType=map
+	// +listMapKey=name
+	BackendSets []NLBBackendSet `json:"backendSets,omitempty"`
+
 	// BackendSetDetails specifies the configuration of a network load balancer backend set.
+	// Deprecated: use `backendSets` instead.
 	// +optional
 	BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"`
 
@@ -1000,7 +1029,25 @@ type NLBSpec struct {
 	ReservedIpIds []string `json:"reservedIpIds,omitempty"`
 }
 
-// BackendSetDetails specifies the configuration of a network load balancer backend set.
+// NLBBackendSet specifies the configuration for a named API server backend set.
+// The type name is historical and applies to both load balancer implementations.
+type NLBBackendSet struct {
+	// Name is the API server backend set identifier.
+	Name string `json:"name"`
+
+	// ListenerPort is the load balancer listener port routed to this backend set.
+	// When omitted, the cluster API server port is used. Explicit values are restricted
+	// to the supported API server listener ports.
+	// +optional
+	ListenerPort *int32 `json:"listenerPort,omitempty"`
+
+	// BackendSetDetails specifies the backend set behavior.
+	// +optional
+	BackendSetDetails BackendSetDetails `json:"backendSetDetails,omitempty"`
+}
+
+// BackendSetDetails specifies the configuration of an API server backend set.
+// The fields cover shared behavior used by both load balancer implementations.
 type BackendSetDetails struct {
 	// If this parameter is enabled, then the network load balancer preserves the source IP of the packet when it is forwarded to backends.
 	// Backends see the original source IP. If the isPreserveSourceDestination parameter is enabled for the network load balancer resource, then this parameter cannot be disabled.
@@ -1031,6 +1078,14 @@ type HealthChecker struct {
 	UrlPath *string `json:"urlPath,omitempty"`
 }
 
+// CanonicalBackendSets returns the canonical API server backend set list.
+// Precedence:
+// 1. If `backendSets` is configured, it is used as-is.
+// 2. Otherwise, a single backend set named APIServerLBBackendSetName is synthesized from the legacy `backendSetDetails`.
+func (in *NLBSpec) CanonicalBackendSets() []NLBBackendSet {
+	return canonicalAPIServerBackendSets(in.BackendSets, in.BackendSetDetails)
+}
+
 // NetworkSpec specifies what the OCI networking resources should look like.
 type NetworkSpec struct {
 	// SkipNetworkManagement defines if the networking spec(VCN related) specified by the user needs to be reconciled(actioned-upon)