diff --git a/upi/openstack/01_security-groups.yaml b/upi/openstack/01_security-groups.yaml index 532bac9ed7e..cdab4bfcb68 100644 --- a/upi/openstack/01_security-groups.yaml +++ b/upi/openstack/01_security-groups.yaml @@ -31,6 +31,34 @@ security_group: "{{ os_sg_master }}" protocol: icmp + - name: 'Create master-sg rule for all TCP ports' + os_security_group_rule: + security_group: "{{ os_sg_master }}" + protocol: tcp + remote_ip_prefix: "{{ cluster_network_cidrs }}" + when: os_networking_type == "CiscoACI" + + - name: 'Create master-sg rule for all UDP ports' + os_security_group_rule: + security_group: "{{ os_sg_master }}" + protocol: udp + remote_ip_prefix: "{{ cluster_network_cidrs }}" + when: os_networking_type == "CiscoACI" + + - name: 'Create worker-sg rule for all TCP ports' + os_security_group_rule: + security_group: "{{ os_sg_worker }}" + protocol: tcp + remote_ip_prefix: "{{ cluster_network_cidrs }}" + when: os_networking_type == "CiscoACI" + + - name: 'Create worker-sg rule for all UDP ports' + os_security_group_rule: + security_group: "{{ os_sg_worker }}" + protocol: udp + remote_ip_prefix: "{{ cluster_network_cidrs }}" + when: os_networking_type == "CiscoACI" + - name: 'Create master-sg rule "machine config server"' os_security_group_rule: security_group: "{{ os_sg_master }}" diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml new file mode 100644 index 00000000000..0718a9fcaf3 --- /dev/null +++ b/upi/openstack/021_network.yaml @@ -0,0 +1,43 @@ +- import_playbook: common.yaml + +- hosts: all + gather_facts: no + + tasks: + - name: 'Create ACI containers node network' + command: + cmd: "neutron net-create {{ os_aci_containers_network }} --apic:nested-domain-name openshift-domain --apic:nested-domain-type openshift --apic:nested_domain_infra_vlan {{ aci_cni['infra_vlan'] }} --apic:nested_domain_service_vlan {{ aci_cni['service_vlan'] }}" + when: os_networking_type == "CiscoACI" + + - name: 'Set the ACI containers cluster network tag' + command: + cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_aci_containers_network }}" + when: os_networking_type == "CiscoACI" + + - name: 'Create the ACI containers subnet' + os_subnet: + name: "{{ os_aci_containers_subnet }}" + network_name: "{{ os_aci_containers_network }}" + no_gateway_ip: yes + cidr: "{{ aci_cni['network_interfaces']['opflex']['subnet'] }}" + allocation_pool_start: "{{ aci_cni['network_interfaces']['opflex']['subnet'] | next_nth_usable(10) }}" + allocation_pool_end: "{{ aci_cni['network_interfaces']['opflex']['subnet'] | ipaddr('last_usable') }}" + when: os_networking_type == "CiscoACI" + + - name: 'Set the ACI containers cluster subnet tag' + command: + cmd: "openstack subnet set --tag {{ cluster_id_tag }} {{ os_aci_containers_subnet }}" + when: os_networking_type == "CiscoACI" + + - name: 'Set MTU for the ACI containers network' + command: + cmd: "openstack network set {{ os_aci_containers_network }} --mtu {{ aci_cni['network_interfaces']['opflex']['mtu'] }}" + when: os_networking_type == "CiscoACI" + + - name: 'Set dns nameserver' + command: + cmd: "openstack subnet set --dns-nameserver {{ item }} {{ os_subnet }}" + when: + - os_networking_type == "CiscoACI" + - aci_cni.dns_ips is defined and aci_cni.dns_ips | length > 0 + with_items: "{{ aci_cni.dns_ips }}" diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index f92c94d5e01..a808ab98e73 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -14,11 +14,21 @@ - name: 'Create the cluster network' os_network: name: "{{ os_network }}" + when: os_networking_type != "CiscoACI" + + - name: 'Create the cluster network with aci-containers-nodes EPG contract relationship for node network' + command: + cmd: "neutron net-create {{ os_network }} --apic:epg_contract_masters list=true type=dict app_profile_name={{ aci_cni['app_profile'] }},name={{ aci_cni['node_epg'] }} --apic:distinguished_names type=dict BridgeDomain={{ aci_cni['network_interfaces']['node']['bd'] }}" + when: os_networking_type == "CiscoACI" - name: 'Set the cluster network tag' command: cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_network }}" + - name: 'Set the primaryClusterNetwork tag' + command: + cmd: "openstack network set --tag {{ os_network }} {{ os_network }}" + - name: 'Create a subnet' os_subnet: name: "{{ os_subnet }}" @@ -26,6 +36,22 @@ cidr: "{{ os_subnet_range }}" allocation_pool_start: "{{ os_subnet_range | next_nth_usable(10) }}" allocation_pool_end: "{{ os_subnet_range | ipaddr('last_usable') }}" + when: os_networking_type != "CiscoACI" + + - name: 'Create the cluster address-scope' + command: + cmd: "neutron address-scope-create node_network_address_scope 4 --apic:distinguished_names type=dict VRF={{ aci_cni['network_interfaces']['node']['vrf'] }}" + when: os_networking_type == "CiscoACI" + + - name: 'Create the subnetpool' + command: + cmd: "neutron subnetpool-create --pool-prefix {{ os_subnet_range }} --address-scope node_network_address_scope node_network_subnet_pool" + when: os_networking_type == "CiscoACI" + + - name: 'Create a subnet' + command: + cmd: "openstack subnet create --network {{ os_network }} --subnet-pool node_network_subnet_pool --subnet-range {{ os_subnet_range }} --allocation-pool start={{ os_subnet_range | next_nth_usable(10) }},end={{ os_subnet_range | ipaddr('last_usable') }} --dhcp {{ os_subnet }}" + when: os_networking_type == "CiscoACI" - name: 'Set the cluster subnet tag' command: @@ -151,3 +177,8 @@ - name: 'Attach the Ingress floating IP to Ingress port' command: cmd: "openstack floating ip set --port {{ os_port_ingress }} {{ os_ingress_fip }}" + + - name: 'Set MTU for the node network' + command: + cmd: "openstack network set {{ os_network }} --mtu {{ aci_cni['network_interfaces']['node']['mtu'] }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/03_bootstrap.yaml b/upi/openstack/03_bootstrap.yaml index aa094ec463e..929bdf800de 100644 --- a/upi/openstack/03_bootstrap.yaml +++ b/upi/openstack/03_bootstrap.yaml @@ -25,6 +25,28 @@ command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap }}" + - name: 'Create the ACI containers bootstrap server port' + os_port: + name: "{{ os_aci_containers_port_bootstrap }}" + network: "{{ os_aci_containers_network }}" + when: os_networking_type == "CiscoACI" + + - name: 'Set ACI containers bootstrap port tag' + command: + cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_aci_containers_port_bootstrap }}" + when: os_networking_type == "CiscoACI" + + - name: 'Create the bootstrap server' + os_server: + name: "{{ os_bootstrap_server_name }}" + image: "{{ os_image_rhcos }}" + flavor: "{{ os_flavor_master }}" + userdata: "{{ lookup('file', os_bootstrap_ignition) | string }}" + auto_ip: no + nics: + - port-name: "{{ os_port_bootstrap }}" + when: os_networking_type != "CiscoACI" + - name: 'Create the bootstrap server' os_server: name: "{{ os_bootstrap_server_name }}" @@ -34,9 +56,12 @@ auto_ip: no nics: - port-name: "{{ os_port_bootstrap }}" + - port-name: "{{ os_aci_containers_port_bootstrap }}" + when: os_networking_type == "CiscoACI" - name: 'Create the bootstrap floating IP' os_floating_ip: state: present + nat_destination: "{{ os_network }}" network: "{{ os_external_network }}" server: "{{ os_bootstrap_server_name }}" diff --git a/upi/openstack/04_control-plane.yaml b/upi/openstack/04_control-plane.yaml index 60eb8214499..146fa8d0fe7 100644 --- a/upi/openstack/04_control-plane.yaml +++ b/upi/openstack/04_control-plane.yaml @@ -29,6 +29,20 @@ cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}" + - name: 'Create the ACI containers Control Plane ports' + os_port: + name: "{{ item.1 }}-{{ item.0 }}" + network: "{{ os_aci_containers_network }}" + with_indexed_items: "{{ [os_aci_containers_port_master] * os_cp_nodes_number }}" + register: ports + when: os_networking_type == "CiscoACI" + + - name: 'Set ACI containers Control Plane ports tag' + command: + cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}" + with_indexed_items: "{{ [os_aci_containers_port_master] * os_cp_nodes_number }}" + when: os_networking_type == "CiscoACI" + - name: 'List the Control Plane Trunks' command: cmd: "openstack network trunk list" @@ -43,6 +57,23 @@ - os_networking_type == "Kuryr" - "os_cp_trunk_name|string not in control_plane_trunks.stdout" + - name: 'Create the Control Plane servers' + os_server: + name: "{{ item.1 }}-{{ item.0 }}" + image: "{{ os_image_rhcos }}" + flavor: "{{ os_flavor_master }}" + auto_ip: no + # The ignition filename will be concatenated with the Control Plane node + # name and its 0-indexed serial number. + # In this case, the first node will look for this filename: + # "{{ infraID }}-master-0-ignition.json" + userdata: "{{ lookup('file', [item.1, item.0, 'ignition.json'] | join('-')) | string }}" + nics: + - port-name: "{{ os_port_master }}-{{ item.0 }}" + - port-name: "{{ os_aci_containers_port_master }}-{{ item.0 }}" + with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" + when: os_networking_type == "CiscoACI" + - name: 'Create the Control Plane servers' os_server: name: "{{ item.1 }}-{{ item.0 }}" @@ -57,3 +88,4 @@ nics: - port-name: "{{ os_port_master }}-{{ item.0 }}" with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" + when: os_networking_type != "CiscoACI" diff --git a/upi/openstack/05_compute-nodes.yaml b/upi/openstack/05_compute-nodes.yaml index 2a3e7f7d0c4..da3ff50427e 100644 --- a/upi/openstack/05_compute-nodes.yaml +++ b/upi/openstack/05_compute-nodes.yaml @@ -27,6 +27,20 @@ cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}" + - name: 'Create the ACI containers Compute ports' + os_port: + name: "{{ item.1 }}-{{ item.0 }}" + network: "{{ os_aci_containers_network }}" + with_indexed_items: "{{ [os_aci_containers_port_worker] * os_compute_nodes_number }}" + register: ports + when: os_networking_type == "CiscoACI" + + - name: 'Set ACI containers Compute ports tag' + command: + cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}" + with_indexed_items: "{{ [os_aci_containers_port_worker] * os_compute_nodes_number }}" + when: os_networking_type == "CiscoACI" + - name: 'List the Compute Trunks' command: cmd: "openstack network trunk list" @@ -41,6 +55,23 @@ - os_networking_type == "Kuryr" - "os_compute_trunk_name|string not in compute_trunks.stdout" + - name: 'Create the Compute servers' + os_server: + name: "{{ item.1 }}-{{ item.0 }}" + image: "{{ os_image_rhcos }}" + flavor: "{{ os_flavor_worker }}" + auto_ip: no + # The ignition filename will be concatenated with the Compute node + # name and its 0-indexed serial number. + # In this case, the first node will look for this filename: + # "{{ infraID }}-worker-0-ignition.json" + userdata: "{{ lookup('file', [item.1, item.0, 'ignition.json'] | join('-')) | string }}" + nics: + - port-name: "{{ os_port_worker }}-{{ item.0 }}" + - port-name: "{{ os_aci_containers_port_worker }}-{{ item.0 }}" + with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" + when: os_networking_type == "CiscoACI" + - name: 'Create the Compute servers' os_server: name: "{{ item.1 }}-{{ item.0 }}" @@ -51,3 +82,4 @@ nics: - port-name: "{{ os_port_worker }}-{{ item.0 }}" with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" + when: os_networking_type != "CiscoACI" diff --git a/upi/openstack/cluster_snat_policy.yaml b/upi/openstack/cluster_snat_policy.yaml new file mode 100644 index 00000000000..733304d858e --- /dev/null +++ b/upi/openstack/cluster_snat_policy.yaml @@ -0,0 +1,10 @@ +- hosts: all + gather_facts: no + tasks: + + - name: 'Create cluster SNAT policy' + k8s: + state: present + kubeconfig: "{{ aci_cni['kubeconfig'] }}" + definition: "{{ lookup('template', 'cluster_snat_policy.conf.j2') }}" + diff --git a/upi/openstack/common.yaml b/upi/openstack/common.yaml index ab7c1551cc9..e3a92d8dd27 100644 --- a/upi/openstack/common.yaml +++ b/upi/openstack/common.yaml @@ -8,7 +8,7 @@ - name: 'Compute resource names' set_fact: cluster_id_tag: "openshiftClusterID={{ infraID }}" - os_network: "{{ infraID }}-network" + os_network: "{{ infraID }}-primaryClusterNetwork" os_subnet: "{{ infraID }}-nodes" os_router: "{{ infraID }}-external-router" # Port names @@ -35,3 +35,7 @@ os_svc_subnet: "{{ infraID }}-kuryr-service-subnet" # Ignition files os_bootstrap_ignition: "{{ infraID }}-bootstrap-ignition.json" + + - name: 'Import variables for CiscoACI CNI' + include: common_ciscoaci.yaml + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/common_ciscoaci.yaml b/upi/openstack/common_ciscoaci.yaml new file mode 100644 index 00000000000..b35562ac73d --- /dev/null +++ b/upi/openstack/common_ciscoaci.yaml @@ -0,0 +1,7 @@ +- name: 'Compute CiscoACI resource names' + set_fact: + os_aci_containers_network: "{{ infraID }}-secondaryClusterNetwork-acicontainers" + os_aci_containers_subnet: "{{ infraID }}-acicontainers-nodes" + os_aci_containers_port_bootstrap: "{{ infraID }}-acicontainers-bootstrap-port" + os_aci_containers_port_master: "{{ infraID }}-acicontainers-master-port" + os_aci_containers_port_worker: "{{ infraID }}-acicontainers-worker-port" diff --git a/upi/openstack/down-02_network.yaml b/upi/openstack/down-02_network.yaml index 15318284c49..bc88c5343df 100644 --- a/upi/openstack/down-02_network.yaml +++ b/upi/openstack/down-02_network.yaml @@ -68,3 +68,30 @@ when: - os_networking_type == "Kuryr" - pods_subnet_pool.stdout != "" + + - name: 'List the cluster subnet pool' + command: + cmd: "openstack subnet pool list --name node_network_subnet_pool" + when: os_networking_type == "CiscoACI" + register: cisco_subnet_pool + + - name: 'Remove the cluster subnet pool' + command: + cmd: "openstack subnet pool delete node_network_subnet_pool" + when: + - os_networking_type == "CiscoACI" + - cisco_subnet_pool.stdout != "" + + - name: 'List the cluster address-scope' + command: + cmd: "openstack address scope list --name node_network_address_scope" + when: os_networking_type == "CiscoACI" + register: cisco_cluster_address_scope + + - name: 'Remove the cluster address-scope' + command: + cmd: "openstack address scope delete node_network_address_scope" + when: + - os_networking_type == "CiscoACI" + - cisco_cluster_address_scope.stdout != "" + diff --git a/upi/openstack/down-03_bootstrap.yaml b/upi/openstack/down-03_bootstrap.yaml index 387860c3172..fe8c13f23a8 100644 --- a/upi/openstack/down-03_bootstrap.yaml +++ b/upi/openstack/down-03_bootstrap.yaml @@ -19,3 +19,9 @@ os_port: name: "{{ os_port_bootstrap }}" state: absent + + - name: 'Remove the second bootstrap server port' + os_port: + name: "{{ os_aci_containers_port_bootstrap }}" + state: absent + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/down-04_control-plane.yaml b/upi/openstack/down-04_control-plane.yaml index a1ab5644f0f..192d805c90b 100644 --- a/upi/openstack/down-04_control-plane.yaml +++ b/upi/openstack/down-04_control-plane.yaml @@ -35,3 +35,10 @@ name: "{{ item.1 }}-{{ item.0 }}" state: absent with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}" + + - name: 'Remove the ACI containers Control Plane ports' + os_port: + name: "{{ item.1 }}-{{ item.0 }}" + state: absent + with_indexed_items: "{{ [os_aci_containers_port_master] * os_cp_nodes_number }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/down-05_compute-nodes.yaml b/upi/openstack/down-05_compute-nodes.yaml index 803eeda5ee9..7ba9e2405a1 100644 --- a/upi/openstack/down-05_compute-nodes.yaml +++ b/upi/openstack/down-05_compute-nodes.yaml @@ -35,3 +35,10 @@ name: "{{ item.1 }}-{{ item.0 }}" state: absent with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}" + + - name: 'Remove the ACI containers Compute ports' + os_port: + name: "{{ item.1 }}-{{ item.0 }}" + state: absent + with_indexed_items: "{{ [os_aci_containers_port_worker] * os_compute_nodes_number }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/files/ingresscontroller_internal_loadbalancer.yaml b/upi/openstack/files/ingresscontroller_internal_loadbalancer.yaml new file mode 100644 index 00000000000..154517e0dc4 --- /dev/null +++ b/upi/openstack/files/ingresscontroller_internal_loadbalancer.yaml @@ -0,0 +1,10 @@ +apiVersion: operator.openshift.io/v1 +kind: IngressController +metadata: + namespace: openshift-ingress-operator + name: default +spec: + endpointPublishingStrategy: + type: LoadBalancerService + loadBalancer: + scope: Internal diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index 7755d94f0fc..f7f3baea68a 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -1,34 +1,46 @@ all: hosts: localhost: + ####################################################################### + aci_cni: + # This section has ACI CNI specific configuration. + # In case of optional configuration, the default + # values are populated after running update_ign.py. + # Those default values are mentioned here. + acc_provision_tar: /path/aci_deployment.yaml1.tar.gz + kubeconfig: /path/kubeconfig + network_interfaces: + node: + name: ens3 # Optional + mtu: 1500 # Optional + bd: uni/tn-prj_/BD-aci-containers--node-bd # Derived + vrf: uni/tn-/ctx- # Derived + opflex: + mtu: 1500 # Optional + name: ens4 # Optional + subnet: 192.168.208.0/20 # Optional + cluster_snat_policy_ip: # Optional, no default value + dns_ips: [] # Optional, no default value + # The following are derived from acc_provision_tar file + # and populated after running the update_ign.py + app_profile: aci-containers- # Derived + node_epg: aci-containers-nodes # Derived + infra_vlan: 4093 # Derived + service_vlan: 1033 # Derived + ####################################################################### ansible_connection: local - ansible_python_interpreter: "{{ansible_playbook_python}}" - - # User-provided values - os_subnet_range: '10.0.0.0/16' - os_flavor_master: 'm1.xlarge' - os_flavor_worker: 'm1.large' - os_image_rhcos: 'rhcos' - os_external_network: 'external' - # OpenShift API floating IP address - os_api_fip: '203.0.113.23' - # OpenShift Ingress floating IP address - os_ingress_fip: '203.0.113.19' - # Service subnet cidr - svc_subnet_range: '172.30.0.0/16' - os_svc_network_range: '172.30.0.0/15' - # Subnet pool prefixes - cluster_network_cidrs: '10.128.0.0/14' - # Subnet pool prefix length + ansible_python_interpreter: '{{ansible_playbook_python}}' + cluster_network_cidrs: 15.128.0.0/16 host_prefix: '23' - # Name of the SDN. - # Possible values are OpenshiftSDN or Kuryr. - os_networking_type: 'OpenshiftSDN' - - # Number of provisioned Control Plane nodes - # 3 is the minimum number for a fully-functional cluster. + os_api_fip: 60.60.60.6 + os_compute_nodes_number: 2 os_cp_nodes_number: 3 - - # Number of provisioned Compute nodes. - # 3 is the minimum number for a fully-functional cluster. - os_compute_nodes_number: 3 + os_external_network: l3out-2 + os_flavor_master: aci_rhel_huge + os_flavor_worker: aci_rhel_huge + os_image_rhcos: rhcos-4.4 + os_ingress_fip: 60.60.60.8 + os_networking_type: CiscoACI + os_subnet_range: 15.11.0.0/27 + os_svc_network_range: 172.30.0.0/15 + svc_subnet_range: 172.30.0.0/16 diff --git a/upi/openstack/post-install.yaml b/upi/openstack/post-install.yaml new file mode 100644 index 00000000000..25a35483d82 --- /dev/null +++ b/upi/openstack/post-install.yaml @@ -0,0 +1,19 @@ +- hosts: all + gather_facts: no + + tasks: + - name: 'Remove to be replaced default ingress controller' + k8s: + state: absent + kubeconfig: "{{ aci_cni['kubeconfig'] }}" + force: yes + wait: yes + definition: "{{ lookup('file', 'files/ingresscontroller_internal_loadbalancer.yaml') }}" + + - name: 'Replace cluster endpoint strategy to Loadbalancer' + k8s: + state: present + kubeconfig: "{{ aci_cni['kubeconfig'] }}" + force: yes + wait: no + definition: "{{ lookup('file', 'files/ingresscontroller_internal_loadbalancer.yaml') }}" diff --git a/upi/openstack/templates/99_master-networkscripts.yaml b/upi/openstack/templates/99_master-networkscripts.yaml new file mode 100644 index 00000000000..09f04157904 --- /dev/null +++ b/upi/openstack/templates/99_master-networkscripts.yaml @@ -0,0 +1,42 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 02-master-network +spec: + config: + ignition: + config: {} + security: + tls: {} + timeouts: {} + version: 2.2.0 + networkd: {} + passwd: {} + storage: + files: + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_ens3.base64 }} + verification: {} + path: {{ ifcfg_ens3.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_ens4.base64 }} + verification: {} + path: {{ ifcfg_ens4.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_opflex_conn.base64 }} + verification: {} + path: {{ ifcfg_opflex_conn.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ route_opflex_conn.base64 }} + verification: {} + path: {{ route_opflex_conn.path }} + mode: 420 + filesystem: root diff --git a/upi/openstack/templates/99_worker-networkscripts.yaml b/upi/openstack/templates/99_worker-networkscripts.yaml new file mode 100644 index 00000000000..aeaabed7651 --- /dev/null +++ b/upi/openstack/templates/99_worker-networkscripts.yaml @@ -0,0 +1,42 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: worker + name: 02-worker-network +spec: + config: + ignition: + config: {} + security: + tls: {} + timeouts: {} + version: 2.2.0 + networkd: {} + passwd: {} + storage: + files: + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_ens3.base64 }} + verification: {} + path: {{ ifcfg_ens3.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_ens4.base64 }} + verification: {} + path: {{ ifcfg_ens4.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_opflex_conn.base64 }} + verification: {} + path: {{ ifcfg_opflex_conn.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ route_opflex_conn.base64 }} + verification: {} + path: {{ route_opflex_conn.path }} + mode: 420 + filesystem: root diff --git a/upi/openstack/templates/cluster_snat_policy.conf.j2 b/upi/openstack/templates/cluster_snat_policy.conf.j2 new file mode 100644 index 00000000000..a77a8f32e03 --- /dev/null +++ b/upi/openstack/templates/cluster_snat_policy.conf.j2 @@ -0,0 +1,7 @@ +apiVersion: aci.snat/v1 +kind: SnatPolicy +metadata: + name: installerclusterdefault +spec: + snatIp: + - {{ aci_cni['cluster_snat_policy_ip'] }} diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py new file mode 100644 index 00000000000..db616fceb08 --- /dev/null +++ b/upi/openstack/update_ign.py @@ -0,0 +1,360 @@ +import base64 +import json +import os +import shutil +import tarfile +import yaml +from jinja2 import Environment, FileSystemLoader + + +#The script does the following things: +#Update boostrap.ign with hostname and CA certs and with additional network-scripts. +#According to the number of the master count, create the JSON files, and add hostname/network-scripts. +#According to the number of the worker count, create the JSON files, and add hostname/network-scripts. + +# Read inventory.yaml for CiscoACI CNI variable +original_inventory = processed_inventory = "inventory.yaml" +with open(original_inventory, 'r') as stream: + try: + localhost = yaml.safe_load(stream)['all']['hosts']['localhost'] + inventory = localhost['aci_cni'] + except yaml.YAMLError as exc: + print(exc) + +# Get accprovision tar path from inventory +try: + acc_provision_tar = inventory['acc_provision_tar'] + os_subnet_range = localhost['os_subnet_range'] +except: + print("inventory.yaml should have acc_provision_tar and os_subnet_range fields") + +# Read acc-provision for vlan values +extract_to = './accProvisionTar' +tar = tarfile.open(acc_provision_tar, "r:gz") +tar.extractall(extract_to) +tar.close() + +data = '' +for filename in os.listdir(extract_to): + if 'ConfigMap-aci-containers-config' in filename: + filepath = "%s/%s" % (extract_to, filename) + with open(filepath, 'r') as stream: + try: + data = yaml.safe_load(stream)['data']['host-agent-config'] + except yaml.YAMLError as exc: + print(exc) + with open(filepath, 'r') as stream: + try: + data_controller = yaml.safe_load(stream)['data']['controller-config'] + except yaml.YAMLError as exc: + print(exc) + +# Extract host-agent-config and obtain vlan values +try: + host_agent_data = json.loads(data) + aci_infra_vlan = host_agent_data['aci-infra-vlan'] + service_vlan = host_agent_data['service-vlan'] + app_profile = host_agent_data['app-profile'] + + controller_data = json.loads(data_controller) + aci_vrf_dn = controller_data['aci-vrf-dn'] + aci_nodebd_dn = controller_data['aci-nodebd-dn'] +except: + print("Couldn't extract host-agent-config from aci-containers ConfigMap") + +# Delete acc_provisionTar that was untarred previously +try: + shutil.rmtree(extract_to) +except OSError as e: + print ("Error: %s - %s." % (e.filename, e.strerror)) + +# Set infra_vlan field in inventory.yaml using accprovision tar value +try: + with open(original_inventory, 'r') as stream: + cur_yaml = yaml.safe_load(stream) + + cur_yaml['all']['hosts']['localhost']['aci_cni']['app_profile'] = app_profile + cur_yaml['all']['hosts']['localhost']['aci_cni']['infra_vlan'] = aci_infra_vlan + + if 'node_epg' not in inventory: + cur_yaml['all']['hosts']['localhost']['aci_cni']['node_epg'] = "aci-containers-nodes" + + cur_yaml['all']['hosts']['localhost']['aci_cni']['service_vlan'] = service_vlan + + if 'network_interfaces' not in inventory: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces'] = dict() + + if 'opflex' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex'] = dict() + + if 'node' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node'] = dict() + + if 'mtu' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['mtu'] = 1500 + + if 'mtu' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['mtu'] = 1500 + + if 'name' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['name'] = 'ens4' + + if 'name' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['name'] = 'ens3' + + if 'subnet' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['subnet'] = '192.168.208.0/20' + + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['vrf'] = aci_vrf_dn + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['bd'] = aci_nodebd_dn + + if cur_yaml: + with open(processed_inventory,'w') as yamlfile: + yaml.safe_dump(cur_yaml, yamlfile) +except: + print("Unable to edit inventory.yaml") +try: + node_interface = cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['name'] + opflex_interface = cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['name'] + master_count = localhost['os_cp_nodes_number'] + worker_count = localhost['os_compute_nodes_number'] +except: + print("Relevant Fields are missing from inventory.yaml ") + +infra_vlan = str(aci_infra_vlan) +infra_id = os.environ.get('INFRA_ID', 'openshift').encode() +neutron_network_mtu = str(cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['mtu']) + +def update(hostname,ignition): + + config_data = {} + + ifcfg_ens3 = ("""TYPE=Ethernet +DEVICE=""" + node_interface + """ +ONBOOT=yes +BOOTPROTO=dhcp +DEFROUTE=yes +PROXY_METHOD=none +BROWSER_ONLY=no +MTU=""" + neutron_network_mtu + """ +IPV4_FAILURE_FATAL=no +IPV6INIT=no +ETHTOOL_OPTS="-K ens3 tx-checksum-ip-generic off" +NAME="System ens3" +UUID=21d47e65-8523-1a06-af22-6f121086f085 +""").encode() + + ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip() + + config_data['ifcfg_ens3'] = {'base64': ifcfg_ens3_b64, 'path': '/etc/sysconfig/network-scripts/ifcfg-ens3'} + + ifcfg_ens4 = ("""TYPE=Ethernet +DEVICE=""" + opflex_interface + """ +ONBOOT=yes +BOOTPROTO=dhcp +DEFROUTE=no +PROXY_METHOD=none +BROWSER_ONLY=no +MTU=""" + neutron_network_mtu + """ +IPV4_FAILURE_FATAL=no +IPV6INIT=no +ETHTOOL_OPTS="-K ens4 tx-checksum-ip-generic off" +NAME="System ens4" +UUID=e27f182b-d125-2c43-5a30-43524d0229ac +""").encode() + + ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() + + config_data['ifcfg_ens4'] = {'base64': ifcfg_ens4_b64, 'path': '/etc/sysconfig/network-scripts/ifcfg-ens4'} + + opflex_conn = ("""VLAN=yes +TYPE=Vlan +PHYSDEV=""" + opflex_interface + """ +VLAN_ID=""" + infra_vlan + """ +REORDER_HDR=yes +GVRP=no +MVRP=no +PROXY_METHOD=none +BROWSER_ONLY=no +BOOTPROTO=dhcp +DEFROUTE=no +IPV4_FAILURE_FATAL=no +IPV6INIT=no +NAME=opflex-conn +DEVICE=""" + opflex_interface + """.""" + infra_vlan + """ +ONBOOT=yes +MTU=""" + neutron_network_mtu + """ +HWADDR= +ETHTOOL_OPTS="-K net0 tx-checksum-ip-generic off" +UUID=eb4377c5-a6d1-f09a-f588-7a6122be32f5 +""").encode() + + ifcfg_opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip() + + config_data['ifcfg_opflex_conn'] = {'base64': ifcfg_opflex_conn_b64, 'path': '/etc/sysconfig/network-scripts/ifcfg-opflex-conn'} + + route_opflex_conn = """ADDRESS0=224.0.0.0 +NETMASK0=240.0.0.0 +METRIC0=1000 +""".encode() + + route_opflex_conn_b64 = base64.standard_b64encode(route_opflex_conn).decode().strip() + + config_data['route_opflex_conn'] = {'base64': route_opflex_conn_b64, 'path': '/etc/sysconfig/network-scripts/route-opflex-conn'} + + files = ignition['storage'].get('files', []) + if 'bootstrap' in hostname.decode(): + ca_cert_path = os.environ.get('OS_CACERT', '') + if ca_cert_path: + with open(ca_cert_path, 'r') as f: + ca_cert = f.read().encode() + ca_cert_b64 = base64.standard_b64encode(ca_cert).decode().strip() + + files.append( + { + 'path': '/opt/openshift/tls/cloud-ca-cert.pem', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + ca_cert_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + # Add master and worker network scripts to bootstrap ignition + env = Environment(loader = FileSystemLoader('./templates'), trim_blocks=True, lstrip_blocks=True) + template_worker = env.get_template('99_worker-networkscripts.yaml') + rendered_worker = template_worker.render(config_data) + worker_b64 = base64.standard_b64encode(rendered_worker.encode()).decode().strip() + + template_master = env.get_template('99_master-networkscripts.yaml') + rendered_master = template_master.render(config_data) + master_b64 = base64.standard_b64encode(rendered_master.encode()).decode().strip() + + files.append( + { + 'path': '/opt/openshift/openshift/99_master-networkscripts.yaml', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + master_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': '/opt/openshift/openshift/99_worker-networkscripts.yaml', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + worker_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + else: + hostname_b64 = base64.standard_b64encode(hostname).decode().strip() + files.append( + { + 'path': '/etc/hostname', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + hostname_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': config_data['ifcfg_ens3']['path'], + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_ens3']['base64'], + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': config_data['ifcfg_ens4']['path'], + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_ens4']['base64'], + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': config_data['ifcfg_opflex_conn']['path'], + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_opflex_conn']['base64'], + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': config_data['route_opflex_conn']['path'], + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['route_opflex_conn']['base64'], + 'verification': {} + }, + 'filesystem': 'root', + }) + + ignition['storage']['files'] = files + return ignition + + +with open('bootstrap.ign', 'r') as f: + ignition = json.load(f) +bootstrap_hostname = infra_id + b'-bootstrap\n' +ignition = update(bootstrap_hostname,ignition) +with open('bootstrap.ign', 'w') as f: + json.dump(ignition, f) + +os.system('cat > ' + infra_id.decode() + '''-bootstrap-ignition.json << EOL +{ + "ignition": { + "config": { + "append": [ + { + "source": "$(swift stat -v | grep StorageURL | awk -F': ' '{print$2}')/bootstrap/bootstrap.ign", + "verification": {} + } + ] + }, + "security": {}, + "timeouts": {}, + "version": "2.2.0" + }, + "networkd": {}, + "passwd": {}, + "storage": {}, + "systemd": {} +} +EOL''') + +for index in range(0,master_count): + master_hostname = infra_id + b'-master-' + str(index).encode() + b'\n' + with open('master.ign', 'r') as f: + ignition = json.load(f) + ignition = update(master_hostname,ignition) + with open(infra_id.decode() + '-master-' + str(index) + '-ignition.json', 'w') as f: + json.dump(ignition, f) + +for index in range(0,worker_count): + master_hostname = infra_id + b'-worker-' + str(index).encode() + b'\n' + with open('worker.ign', 'r') as f: + ignition = json.load(f) + ignition = update(master_hostname,ignition) + with open(infra_id.decode() + '-worker-' + str(index) + '-ignition.json', 'w') as f: + json.dump(ignition, f) diff --git a/upi/openstack/update_machineset.py b/upi/openstack/update_machineset.py new file mode 100644 index 00000000000..107d2fad1b2 --- /dev/null +++ b/upi/openstack/update_machineset.py @@ -0,0 +1,19 @@ +import yaml; +path = "openshift/99_openshift-cluster-api_worker-machineset-0.yaml" +data = yaml.safe_load(open(path)); +networks = data['spec']['template']['spec']['providerSpec']['value'].get('networks', []) +networks.append( + { + 'filter': {}, + 'subnets': [ + { + 'filter': { + 'name': str(data['metadata']['labels']['machine.openshift.io/cluster-api-cluster'] + '-acicontainers-nodes'), + 'tags': str('openshiftClusterID=' + data['metadata']['labels']['machine.openshift.io/cluster-api-cluster']) + } + } + ] + } +) +data['spec']['template']['spec']['providerSpec']['value']['networks'] = networks +open(path, "w").write(yaml.dump(data, default_flow_style=False))