DNM - Jefferson to release-4.4

upi/openstack/01_security-groups.yaml

@@ -31,6 +31,34 @@
       security_group: "{{ os_sg_master }}"
       protocol: icmp
 
+  - name: 'Create master-sg rule for all TCP ports'
+    os_security_group_rule:
+      security_group: "{{ os_sg_master }}"
+      protocol: tcp
+      remote_ip_prefix: "{{ cluster_network_cidrs }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Create master-sg rule for all UDP ports'
+    os_security_group_rule:
+      security_group: "{{ os_sg_master }}"
+      protocol: udp
+      remote_ip_prefix: "{{ cluster_network_cidrs }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Create worker-sg rule for all TCP ports'
+    os_security_group_rule:
+      security_group: "{{ os_sg_worker }}"
+      protocol: tcp
+      remote_ip_prefix: "{{ cluster_network_cidrs }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Create worker-sg rule for all UDP ports'
+    os_security_group_rule:
+      security_group: "{{ os_sg_worker }}"
+      protocol: udp
+      remote_ip_prefix: "{{ cluster_network_cidrs }}"
+    when: os_networking_type == "CiscoACI"
+
   - name: 'Create master-sg rule "machine config server"'
     os_security_group_rule:
       security_group: "{{ os_sg_master }}"

upi/openstack/021_network.yaml

@@ -0,0 +1,43 @@
+- import_playbook: common.yaml
+
+- hosts: all
+  gather_facts: no
+
+  tasks:
+  - name: 'Create ACI containers node network'
+    command:
+      cmd: "neutron net-create {{ os_aci_containers_network }} --apic:nested-domain-name openshift-domain --apic:nested-domain-type openshift --apic:nested_domain_infra_vlan {{ aci_cni['infra_vlan'] }} --apic:nested_domain_service_vlan {{ aci_cni['service_vlan'] }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Set the ACI containers cluster network tag'
+    command:
+      cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_aci_containers_network }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Create the ACI containers subnet'
+    os_subnet:
+      name: "{{ os_aci_containers_subnet }}"
+      network_name: "{{ os_aci_containers_network }}"
+      no_gateway_ip: yes
+      cidr: "{{ aci_cni['network_interfaces']['opflex']['subnet'] }}"
+      allocation_pool_start: "{{ aci_cni['network_interfaces']['opflex']['subnet'] | next_nth_usable(10) }}"
+      allocation_pool_end: "{{ aci_cni['network_interfaces']['opflex']['subnet'] | ipaddr('last_usable') }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Set the ACI containers cluster subnet tag'
+    command:
+      cmd: "openstack subnet set --tag {{ cluster_id_tag }} {{ os_aci_containers_subnet }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Set MTU for the ACI containers network'
+    command:
+      cmd: "openstack network set {{ os_aci_containers_network }} --mtu {{ aci_cni['network_interfaces']['opflex']['mtu'] }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Set dns nameserver'
+    command:
+      cmd: "openstack subnet set --dns-nameserver {{ item }} {{ os_subnet }}"
+    when:
+      - os_networking_type == "CiscoACI"
+      - aci_cni.dns_ips is defined and aci_cni.dns_ips | length > 0
+    with_items: "{{ aci_cni.dns_ips }}"

upi/openstack/02_network.yaml

@@ -14,18 +14,44 @@
   - name: 'Create the cluster network'
     os_network:
       name: "{{ os_network }}"
+    when: os_networking_type != "CiscoACI"
+
+  - name: 'Create the cluster network with aci-containers-nodes EPG contract relationship for node network'
+    command:
+      cmd: "neutron net-create {{ os_network }} --apic:epg_contract_masters list=true type=dict app_profile_name={{ aci_cni['app_profile'] }},name={{ aci_cni['node_epg'] }} --apic:distinguished_names type=dict BridgeDomain={{ aci_cni['network_interfaces']['node']['bd'] }}"
+    when: os_networking_type == "CiscoACI"
 
   - name: 'Set the cluster network tag'
     command:
       cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_network }}"
 
+  - name: 'Set the primaryClusterNetwork tag'
+    command:
+      cmd: "openstack network set --tag {{ os_network }} {{ os_network }}"
+
   - name: 'Create a subnet'
     os_subnet:
       name: "{{ os_subnet }}"
       network_name: "{{ os_network }}"
       cidr: "{{ os_subnet_range }}"
       allocation_pool_start: "{{ os_subnet_range | next_nth_usable(10) }}"
       allocation_pool_end: "{{ os_subnet_range | ipaddr('last_usable') }}"
+    when: os_networking_type != "CiscoACI"
+
+  - name: 'Create the cluster address-scope'
+    command:
+      cmd: "neutron address-scope-create node_network_address_scope 4 --apic:distinguished_names type=dict VRF={{ aci_cni['network_interfaces']['node']['vrf'] }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Create the subnetpool'
+    command:
+      cmd: "neutron subnetpool-create --pool-prefix {{ os_subnet_range }} --address-scope node_network_address_scope node_network_subnet_pool"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Create a subnet'
+    command:
+      cmd: "openstack subnet create --network {{ os_network }}  --subnet-pool node_network_subnet_pool --subnet-range {{ os_subnet_range }} --allocation-pool start={{ os_subnet_range | next_nth_usable(10) }},end={{ os_subnet_range | ipaddr('last_usable') }} --dhcp {{ os_subnet }}"
+    when: os_networking_type == "CiscoACI"
 
   - name: 'Set the cluster subnet tag'
     command:
@@ -151,3 +177,8 @@
   - name: 'Attach the Ingress floating IP to Ingress port'
     command:
       cmd: "openstack floating ip set --port {{ os_port_ingress }} {{ os_ingress_fip }}"
+
+  - name: 'Set MTU for the node network'
+    command:
+      cmd: "openstack network set {{ os_network }} --mtu {{ aci_cni['network_interfaces']['node']['mtu'] }}"
+    when: os_networking_type == "CiscoACI"

upi/openstack/03_bootstrap.yaml

@@ -25,6 +25,28 @@
     command:
       cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap }}"
 
+  - name: 'Create the ACI containers bootstrap server port'
+    os_port:
+      name: "{{ os_aci_containers_port_bootstrap }}"
+      network: "{{ os_aci_containers_network }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Set ACI containers bootstrap port tag'
+    command:
+      cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_aci_containers_port_bootstrap }}"
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Create the bootstrap server'
+    os_server:
+      name: "{{ os_bootstrap_server_name }}"
+      image: "{{ os_image_rhcos }}"
+      flavor: "{{ os_flavor_master }}"
+      userdata: "{{ lookup('file', os_bootstrap_ignition) | string }}"
+      auto_ip: no
+      nics:
+      - port-name: "{{ os_port_bootstrap }}"
+    when: os_networking_type != "CiscoACI"
+
   - name: 'Create the bootstrap server'
     os_server:
       name: "{{ os_bootstrap_server_name }}"
@@ -34,9 +56,12 @@
       auto_ip: no
       nics:
       - port-name: "{{ os_port_bootstrap }}"
+      - port-name: "{{ os_aci_containers_port_bootstrap }}"
+    when: os_networking_type == "CiscoACI"
 
   - name: 'Create the bootstrap floating IP'
     os_floating_ip:
       state: present
+      nat_destination: "{{ os_network }}"
       network: "{{ os_external_network }}"
       server: "{{ os_bootstrap_server_name }}"

upi/openstack/04_control-plane.yaml

@@ -29,6 +29,20 @@
       cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}"
     with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}"
 
+  - name: 'Create the ACI containers Control Plane ports'
+    os_port:
+      name: "{{ item.1 }}-{{ item.0 }}"
+      network: "{{ os_aci_containers_network }}"
+    with_indexed_items: "{{ [os_aci_containers_port_master] * os_cp_nodes_number }}"
+    register: ports
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Set ACI containers Control Plane ports tag'
+    command:
+      cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}"
+    with_indexed_items: "{{ [os_aci_containers_port_master] * os_cp_nodes_number }}"
+    when: os_networking_type == "CiscoACI"
+
   - name: 'List the Control Plane Trunks'
     command:
       cmd: "openstack network trunk list"
@@ -43,6 +57,23 @@
     - os_networking_type == "Kuryr"
     - "os_cp_trunk_name|string not in control_plane_trunks.stdout"
 
+  - name: 'Create the Control Plane servers'
+    os_server:
+      name: "{{ item.1 }}-{{ item.0 }}"
+      image: "{{ os_image_rhcos }}"
+      flavor: "{{ os_flavor_master }}"
+      auto_ip: no
+      # The ignition filename will be concatenated with the Control Plane node
+      # name and its 0-indexed serial number.
+      # In this case, the first node will look for this filename:
+      #    "{{ infraID }}-master-0-ignition.json"
+      userdata: "{{ lookup('file', [item.1, item.0, 'ignition.json'] | join('-')) | string }}"
+      nics:
+      - port-name: "{{ os_port_master }}-{{ item.0 }}"
+      - port-name: "{{ os_aci_containers_port_master }}-{{ item.0 }}"
+    with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}"
+    when: os_networking_type == "CiscoACI"
+
   - name: 'Create the Control Plane servers'
     os_server:
       name: "{{ item.1 }}-{{ item.0 }}"
@@ -57,3 +88,4 @@
       nics:
       - port-name: "{{ os_port_master }}-{{ item.0 }}"
     with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}"
+    when: os_networking_type != "CiscoACI"

upi/openstack/05_compute-nodes.yaml

@@ -27,6 +27,20 @@
       cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}"
     with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}"
 
+  - name: 'Create the ACI containers Compute ports'
+    os_port:
+      name: "{{ item.1 }}-{{ item.0 }}"
+      network: "{{ os_aci_containers_network }}"
+    with_indexed_items: "{{ [os_aci_containers_port_worker] * os_compute_nodes_number }}"
+    register: ports
+    when: os_networking_type == "CiscoACI"
+
+  - name: 'Set ACI containers Compute ports tag'
+    command:
+      cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}"
+    with_indexed_items: "{{ [os_aci_containers_port_worker] * os_compute_nodes_number }}"
+    when: os_networking_type == "CiscoACI"
+
   - name: 'List the Compute Trunks'
     command:
       cmd: "openstack network trunk list"
@@ -41,6 +55,23 @@
     - os_networking_type == "Kuryr"
     - "os_compute_trunk_name|string not in compute_trunks.stdout"
 
+  - name: 'Create the Compute servers'
+    os_server:
+      name: "{{ item.1 }}-{{ item.0 }}"
+      image: "{{ os_image_rhcos }}"
+      flavor: "{{ os_flavor_worker }}"
+      auto_ip: no
+      # The ignition filename will be concatenated with the Compute node
+      # name and its 0-indexed serial number.
+      # In this case, the first node will look for this filename:
+      #    "{{ infraID }}-worker-0-ignition.json"
+      userdata: "{{ lookup('file', [item.1, item.0, 'ignition.json'] | join('-')) | string }}"
+      nics:
+      - port-name: "{{ os_port_worker }}-{{ item.0 }}"
+      - port-name: "{{ os_aci_containers_port_worker }}-{{ item.0 }}"
+    with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}"
+    when: os_networking_type == "CiscoACI"
+
   - name: 'Create the Compute servers'
     os_server:
       name: "{{ item.1 }}-{{ item.0 }}"
@@ -51,3 +82,4 @@
       nics:
       - port-name: "{{ os_port_worker }}-{{ item.0 }}"
     with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}"
+    when: os_networking_type != "CiscoACI"

upi/openstack/cluster_snat_policy.yaml

@@ -0,0 +1,10 @@
+- hosts: all
+  gather_facts: no
+  tasks:
+
+  - name: 'Create cluster SNAT policy'
+    k8s:
+      state: present
+      kubeconfig: "{{ aci_cni['kubeconfig'] }}"
+      definition: "{{ lookup('template', 'cluster_snat_policy.conf.j2') }}"
+

upi/openstack/common.yaml

@@ -8,7 +8,7 @@
   - name: 'Compute resource names'
     set_fact:
       cluster_id_tag: "openshiftClusterID={{ infraID }}"
-      os_network: "{{ infraID }}-network"
+      os_network: "{{ infraID }}-primaryClusterNetwork"
       os_subnet: "{{ infraID }}-nodes"
       os_router: "{{ infraID }}-external-router"
       # Port names
@@ -35,3 +35,7 @@
       os_svc_subnet: "{{ infraID }}-kuryr-service-subnet"
       # Ignition files
       os_bootstrap_ignition: "{{ infraID }}-bootstrap-ignition.json"
+
+  - name: 'Import variables for CiscoACI CNI'
+    include: common_ciscoaci.yaml
+    when: os_networking_type == "CiscoACI"

upi/openstack/common_ciscoaci.yaml

@@ -0,0 +1,7 @@
+- name: 'Compute CiscoACI resource names'
+  set_fact:
+    os_aci_containers_network: "{{ infraID }}-secondaryClusterNetwork-acicontainers"
+    os_aci_containers_subnet: "{{ infraID }}-acicontainers-nodes"
+    os_aci_containers_port_bootstrap: "{{ infraID }}-acicontainers-bootstrap-port"
+    os_aci_containers_port_master: "{{ infraID }}-acicontainers-master-port"
+    os_aci_containers_port_worker: "{{ infraID }}-acicontainers-worker-port"

upi/openstack/down-02_network.yaml

@@ -68,3 +68,30 @@
     when:
     - os_networking_type == "Kuryr"
     - pods_subnet_pool.stdout != ""
+
+  - name: 'List the cluster subnet pool'
+    command:
+      cmd: "openstack subnet pool list --name node_network_subnet_pool"
+    when: os_networking_type == "CiscoACI"
+    register: cisco_subnet_pool
+
+  - name: 'Remove the cluster subnet pool'
+    command:
+      cmd: "openstack subnet pool delete node_network_subnet_pool"
+    when:
+      - os_networking_type == "CiscoACI"
+      - cisco_subnet_pool.stdout != ""
+
+  - name: 'List the cluster address-scope'
+    command:
+      cmd: "openstack address scope list --name node_network_address_scope"
+    when: os_networking_type == "CiscoACI"
+    register: cisco_cluster_address_scope
+
+  - name: 'Remove the cluster address-scope'
+    command:
+      cmd: "openstack address scope delete node_network_address_scope"
+    when:
+      - os_networking_type == "CiscoACI"
+      - cisco_cluster_address_scope.stdout != ""
+

upi/openstack/down-03_bootstrap.yaml

@@ -19,3 +19,9 @@
     os_port:
       name: "{{ os_port_bootstrap }}"
       state: absent
+
+  - name: 'Remove the second bootstrap server port'
+    os_port:
+      name: "{{ os_aci_containers_port_bootstrap }}"
+      state: absent
+    when: os_networking_type == "CiscoACI"

upi/openstack/down-04_control-plane.yaml

@@ -35,3 +35,10 @@
       name: "{{ item.1 }}-{{ item.0 }}"
       state: absent
     with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}"
+
+  - name: 'Remove the ACI containers Control Plane ports'
+    os_port:
+      name: "{{ item.1 }}-{{ item.0 }}"
+      state: absent
+    with_indexed_items: "{{ [os_aci_containers_port_master] * os_cp_nodes_number }}"
+    when: os_networking_type == "CiscoACI"

upi/openstack/down-05_compute-nodes.yaml

@@ -35,3 +35,10 @@
       name: "{{ item.1 }}-{{ item.0 }}"
       state: absent
     with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}"
+
+  - name: 'Remove the ACI containers Compute ports'
+    os_port:
+      name: "{{ item.1 }}-{{ item.0 }}"
+      state: absent
+    with_indexed_items: "{{ [os_aci_containers_port_worker] * os_compute_nodes_number }}"
+    when: os_networking_type == "CiscoACI"

upi/openstack/files/ingresscontroller_internal_loadbalancer.yaml

@@ -0,0 +1,10 @@
+apiVersion: operator.openshift.io/v1
+kind: IngressController
+metadata: 
+  namespace: openshift-ingress-operator
+  name: default
+spec: 
+  endpointPublishingStrategy:
+     type: LoadBalancerService
+     loadBalancer: 
+        scope: Internal