Merge pull request #69 from pfl/dranet

Dranet

Author: tkatila

Makefile

@@ -152,7 +152,7 @@ BUILD_DATE = $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 
 GO111MODULE=on
 CGOFLAGS=-trimpath -tags osusergo,netgo
-GCFLAGS=all=-spectre=all -N -l
+GCFLAGS=all=-spectre=all
 ASMFLAGS=all=-spectre=all
 LDFLAGS=all=-s -w -X ${PKG}/internal/version.gitInfo=${GIT_INFO} -X ${PKG}/internal/version.buildDate=${BUILD_DATE}
 

README.md

@@ -2,15 +2,14 @@
 
 [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/intel/network-operator/badge)](https://scorecard.dev/viewer/?uri=github.com/intel/network-operator)
 
-CAUTION: This is an beta / non-production software, do not use on production clusters.
-
-Network Operator allows automatic configuring and easier use of RDMA NICs with Intel AI accelerators.
+Network Operator allows automatic configuring and easier use of NICs with Intel AI accelerators.
 
 ## Description
 
-Network operator currently supports Gaudi and its integrated scale-out network interfaces.
+Network operator supports Gaudi and its integrated scale-out network interfaces
+as well as host based network interfaces.
 
-### Intel® Gaudi®
+### Intel® Gaudi® integrated NICs
 
 Intel Gaudi and its integrated NICs are supported in two modes: L2 and L3.
 
@@ -28,16 +27,23 @@ The operator will deploy configuration Pods to the worker nodes which will liste
 
 More info on the switch topology and configurations is available [here](https://docs.habana.ai/en/v1.20.0/Management_and_Monitoring/Network_Configuration/Configure_E2E_Test_in_L3.html).
 
+### Host based network interface cards
+
+Network operator uses [DRANet](https://github.com/kubernetes-sigs/dranet) to configure
+host based network interface cards. Currently network cards supporting RDMA are
+requested with the [DRANet DeviceClass](config/deployments/dranet/deviceclass.yaml)
+with the name of the DeviceClass being configurable in the
+[HostNicScaleOutSpec](api/v1alpha1/networkconfiguration_types.go).
+
 ### Future work
 
-* Enable Host-NIC use in cluster
 * Support to install Host-NIC KMD
 * Configure RDMA NICs to be used with Intel AI accelerators
 
 ### Dependencies
 
 The operator depends on following Kubernetes components:
-* Intel Gaudi base operator
+* Intel Gaudi base operator (Gaudi accelerators)
 * Node Feature Discovery
 * Cert-manager
 * Prometheus (optional)
@@ -50,7 +56,7 @@ The operator depends on following Kubernetes components:
 - kubectl version v1.31+.
 - Access to a Kubernetes v1.31+ cluster.
 
-### Deploy operator using kubectl
+### Deploy operator with Gaudi support using kubectl
 
 Images are available at [dockerhub.io](https://hub.docker.com/r/intel/intel-network-operator).
 
@@ -76,6 +82,20 @@ sample with:
 kubectl apply -f config/operator/samples/gaudi-l3.yaml
 ```
 
+### Deploy operator with host based NIC support
+
+**Install operator into the cluster:**
+
+```sh
+kubectl apply -k config/operator/default/
+```
+
+**Create instance for host NIC**
+
+```sh
+kubectl apply -f config/operator/samples/hostnic-so.yaml
+```
+
 ### Remove operator using kubectl
 
 **Delete the instances (CRs) from the cluster:**
@@ -84,6 +104,12 @@ kubectl apply -f config/operator/samples/gaudi-l3.yaml
 kubectl delete -f config/operator/samples/gaudi-l3.yaml
 ```
 
+**OR**
+
+```sh
+kubectl delete -f config/operator/samples/hostnic-so.yaml
+```
+
 **Uninstall the controller from the cluster:**
 
 ```sh
@@ -99,7 +125,7 @@ kubectl delete -f config/nfd/gaudi-device-rule.yaml
 
 See the [README for Helm installation](charts/network-operator/README.md).
 
-### Prometheus scale-out network metrics
+### Prometheus scale-out network metrics for Gaudi
 
 In order to supply scale-out network metrics to Prometheus, enable them
 in the CR by setting `networkMetrics=true`. Use for example the
@@ -146,6 +172,14 @@ kubectl delete -f config/discovery/prometheus/metrics-service.yaml
 
 The most important Network Operator CRD properties are:
 
+* `configurationType` string
+
+   Enable Gaudi network scale-out configuration with `gaudi-so` or host based NICs with `hostnic-so`.
+
+**Applicable for Gaudi Accelerators**
+
+Properties under `gaudiScaleOut`
+
 * `disableNetworkManager` boolean
 
     Disable Gaudi scale-out interfaces in NetworkManager. For nodes where NetworkManager tries
@@ -175,6 +209,15 @@ The most important Network Operator CRD properties are:
 
     Enable scale-out network metrics from an HTTP endpoint on the Pod. Prometheus can be configured to scrape the endpoint with [Service and ServiceMonitor objects](#prometheus-scale-out-network-metrics).
 
+**Applicable for host NIC**
+
+Properties under `hostNicScaleOut`
+
+* `installDranet` boolean
+
+    Have Network Operator automatically install DRANet in the operator's namespace.
+    If set to `false` it is assumed that the cluster admin already has DRANet set up.
+
 The full set of properties is available in the [NetworkClusterPolicy CRD definition](config/operator/crd/bases/intel.com_networkclusterpolicies.yaml).
 Examples of Network Operator CRDs are found in the [samples directory](config/operator/samples/).
 

api/v1alpha1/networkconfiguration_types.go

@@ -21,19 +21,21 @@ import (
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
 // NetworkClusterPolicySpec defines the desired state of NetworkClusterPolicy
+// +kubebuilder:validation:XValidation:rule="self.configurationType != 'gaudi-so' || has(self.nodeSelector)",message="nodeSelector is required when configurationType is gaudi-so"
 type NetworkClusterPolicySpec struct {
-	// Configuration type that the operator will configure to the nodes. Possible options: gaudi-so.
-	// TODO: plausible other options: host-nic
-	// +kubebuilder:validation:Enum=gaudi-so
+	// Configuration type that the operator will configure to the nodes. Possible options: gaudi-so, hostnic-so.
+	// +kubebuilder:validation:Enum=gaudi-so;hostnic-so
 	ConfigurationType string `json:"configurationType"`
 
 	// Select which nodes the operator should target. Align with labels created by NFD.
-	// +kubebuilder:validation:Required
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
 
 	// Gaudi Scale-Out specific settings. Only valid when configuration type is 'gaudi-so'
 	GaudiScaleOut GaudiScaleOutSpec `json:"gaudiScaleOut,omitempty"`
 
+	// Host NIC Scale-Out specific settings, valid when configuration type is 'hostnic-so'
+	HostNicScaleOut HostNicScaleOutSpec `json:"hostNicScaleOut,omitempty"`
+
 	// LogLevel sets the operator's log level.
 	// +kubebuilder:validation:Minimum=0
 	// +kubebuilder:validation:Maximum=8
@@ -47,7 +49,6 @@ type GaudiScaleOutSpec struct {
 	DisableNetworkManager bool `json:"disableNetworkManager,omitempty"`
 
 	// Layer where the configuration should occur. Possible options: L2 and L3.
-	// +kubebuilder:validation:Required
 	// +kubebuilder:validation:Enum=L2;L3
 	Layer string `json:"layer,omitempty"`
 
@@ -78,6 +79,37 @@ type GaudiScaleOutSpec struct {
 	NetworkMetrics bool `json:"networkMetrics,omitempty"`
 }
 
+// RDMA device specification
+type RDMADeviceClassSpec struct {
+	// Name of the RDMA device class
+	Name string `json:"name"`
+}
+
+// Configuration specific for DRANet
+type DranetSpec struct {
+	// Dranet image to use.
+	Image string `json:"image,omitempty"`
+
+	// Pull policy for the dranet image.
+	// +kubebuilder:validation:Enum=Never;Always;IfNotPresent
+	PullPolicy string `json:"pullPolicy,omitempty"`
+
+	// Device Class for DRANet RDMA resources
+	RDMADeviceClass *RDMADeviceClassSpec `json:"rdmaDeviceClass,omitempty"`
+}
+
+// HostNicScaleOutSpec defines the desired state of HostNicScaleOut and will install
+// dranet, https://github.com/kubernetes-sigs/dranet/, to define interface resources
+type HostNicScaleOutSpec struct {
+	// Have the Network Operator install DRANet into its namespace. If set
+	// to false, the cluster admin is expected to provision DRANet by other
+	// means.
+	InstallDRANet bool `json:"installDranet"`
+
+	// Dranet specification,  not used if installDranet is false
+	Dranet DranetSpec `json:"dranet,omitempty"`
+}
+
 // NetworkClusterPolicyStatus defines the observed state of NetworkClusterPolicy
 type NetworkClusterPolicyStatus struct {
 	Targets    int32    `json:"targets"`

api/v1alpha1/networkconfiguration_webhook.go

@@ -29,7 +29,9 @@ import (
 var netpolicylog = logf.Log.WithName("nicclusterpolicy-resource")
 
 const (
-	gaudiScaleOut = "gaudi-so"
+	gaudiScaleOut          = "gaudi-so"
+	hostNicScaleOut        = "hostnic-so"
+	DefaultRDMADeviceClass = "dranet-rdma"
 )
 
 type emptyNodeSelectorError struct{}
@@ -50,6 +52,12 @@ func (e unknownConfigurationError) Error() string {
 	return "unknown error"
 }
 
+type missingDeviceClassNameError struct{}
+
+func (e missingDeviceClassNameError) Error() string {
+	return "missing device class name"
+}
+
 // SetupWebhookWithManager will setup the manager to manage the webhooks
 func (r *NetworkClusterPolicy) SetupWebhookWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewWebhookManagedBy(mgr).
@@ -70,6 +78,12 @@ func (r *NetworkClusterPolicy) Default() {
 		if len(r.Spec.GaudiScaleOut.Image) == 0 {
 			r.Spec.GaudiScaleOut.Image = "intel/intel-network-linkdiscovery:latest"
 		}
+	case hostNicScaleOut:
+		rdmaDeviceClass := r.Spec.HostNicScaleOut.Dranet.RDMADeviceClass
+		if rdmaDeviceClass != nil && rdmaDeviceClass.Name == "" {
+			r.Spec.HostNicScaleOut.Dranet.RDMADeviceClass.Name = DefaultRDMADeviceClass
+		}
+
 	}
 }
 
@@ -118,14 +132,23 @@ func validateNodeSelector(nodeSelector map[string]string) error {
 	return nil
 }
 
-func validateSpec(s NetworkClusterPolicySpec) (admission.Warnings, error) {
-	if err := validateNodeSelector(s.NodeSelector); err != nil {
-		return nil, err
+func validateHostNicSoSpec(s HostNicScaleOutSpec) error {
+	if s.Dranet.RDMADeviceClass != nil && s.Dranet.RDMADeviceClass.Name == "" {
+		return missingDeviceClassNameError{}
 	}
 
+	return nil
+}
+
+func validateSpec(s NetworkClusterPolicySpec) (admission.Warnings, error) {
 	switch s.ConfigurationType {
 	case gaudiScaleOut:
+		if err := validateNodeSelector(s.NodeSelector); err != nil {
+			return nil, err
+		}
 		return nil, validateGaudiSoSpec(s.GaudiScaleOut)
+	case hostNicScaleOut:
+		return nil, validateHostNicSoSpec(s.HostNicScaleOut)
 	default:
 		return nil, unknownConfigurationError{}
 	}

api/v1alpha1/zz_generated.deepcopy.go

@@ -26,6 +26,7 @@ RUN go mod download
 # Copy the go source
 COPY cmd/operator cmd/operator
 COPY api/ api/
+COPY config config
 COPY config/discovery config/discovery
 COPY internal/controller/ internal/controller/
 COPY internal/version/ internal/version/