From 35f9989aec650f65c2384f779b1d3b4db34e521c Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Mon, 29 Jun 2020 14:22:33 -0700 Subject: [PATCH 01/48] First commits for ACI CNI 1. Create a second neutron network 2. Create an additional port on each hostt 3. Create floating IPs for control plane and compute nodes as well --- upi/openstack/021_network.yaml | 30 ++++++++++++++++++++++++ upi/openstack/03_bootstrap.yaml | 14 +++++++++++ upi/openstack/04_control-plane.yaml | 25 ++++++++++++++++++++ upi/openstack/05_compute-nodes.yaml | 23 ++++++++++++++++++ upi/openstack/common.yaml | 5 ++++ upi/openstack/down-03_bootstrap.yaml | 5 ++++ upi/openstack/down-04_control-plane.yaml | 6 +++++ upi/openstack/down-05_compute-nodes.yaml | 6 +++++ 8 files changed, 114 insertions(+) create mode 100644 upi/openstack/021_network.yaml diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml new file mode 100644 index 00000000000..8f03659c319 --- /dev/null +++ b/upi/openstack/021_network.yaml @@ -0,0 +1,30 @@ +- import_playbook: common.yaml + +- hosts: all + gather_facts: no + + tasks: + - name: 'Create second node network' + command: + cmd: "neutron net-create {{ os_network2 }} --apic:nested-domain-name openshift-domain --apic:nested-domain-type openshift --apic:nested_domain_infra_vlan 4093 --apic:nested_domain_service_vlan 2003" + + - name: 'Set the second cluster network tag' + command: + cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_network2 }}" + + - name: 'Create a subnet2' + os_subnet: + name: "{{ os_subnet2 }}" + network_name: "{{ os_network2 }}" + no_gateway_ip: yes + cidr: "{{ os_subnet_range2 }}" + allocation_pool_start: "{{ os_subnet_range2 | next_nth_usable(10) }}" + allocation_pool_end: "{{ os_subnet_range2 | ipaddr('last_usable') }}" + + - name: 'Set the cluster subnet tag' + command: + cmd: "openstack subnet set --tag {{ cluster_id_tag }} {{ os_subnet2 }}" + + - name: 'Set dns nameserver' + command: + cmd: "openstack subnet set --dns-nameserver {{ dns_ip }} {{ os_subnet }}" diff --git a/upi/openstack/03_bootstrap.yaml b/upi/openstack/03_bootstrap.yaml index aa094ec463e..7fefaa4031a 100644 --- a/upi/openstack/03_bootstrap.yaml +++ b/upi/openstack/03_bootstrap.yaml @@ -25,6 +25,18 @@ command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap }}" + - name: 'Create the bootstrap server port' + os_port: + name: "{{ os_port_bootstrap2 }}" + network: "{{ os_network2 }}" + allowed_address_pairs: + - ip_address: "{{ os_subnet_range | next_nth_usable(5) }}" + - ip_address: "{{ os_subnet_range | next_nth_usable(6) }}" + + - name: 'Set bootstrap port tag 2' + command: + cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap2 }}" + - name: 'Create the bootstrap server' os_server: name: "{{ os_bootstrap_server_name }}" @@ -34,9 +46,11 @@ auto_ip: no nics: - port-name: "{{ os_port_bootstrap }}" + - port-name: "{{ os_port_bootstrap2 }}" - name: 'Create the bootstrap floating IP' os_floating_ip: state: present + nat_destination: "{{ os_network }}" network: "{{ os_external_network }}" server: "{{ os_bootstrap_server_name }}" diff --git a/upi/openstack/04_control-plane.yaml b/upi/openstack/04_control-plane.yaml index 60eb8214499..755fa1e3835 100644 --- a/upi/openstack/04_control-plane.yaml +++ b/upi/openstack/04_control-plane.yaml @@ -29,6 +29,22 @@ cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}" + - name: 'Create the Control Plane ports 2' + os_port: + name: "{{ item.1 }}-{{ item.0 }}" + network: "{{ os_network2 }}" + allowed_address_pairs: + - ip_address: "{{ os_subnet_range2 | next_nth_usable(5) }}" + - ip_address: "{{ os_subnet_range2 | next_nth_usable(6) }}" + - ip_address: "{{ os_subnet_range2 | next_nth_usable(7) }}" + with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" + register: ports + + - name: 'Set Control Plane ports tag2' + command: + cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}" + with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" + - name: 'List the Control Plane Trunks' command: cmd: "openstack network trunk list" @@ -56,4 +72,13 @@ userdata: "{{ lookup('file', [item.1, item.0, 'ignition.json'] | join('-')) | string }}" nics: - port-name: "{{ os_port_master }}-{{ item.0 }}" + - port-name: "{{ os_port_master2 }}-{{ item.0 }}" + with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" + + - name: 'Create the master floating IP' + os_floating_ip: + state: present + nat_destination: "{{ os_network }}" + network: "{{ os_external_network }}" + server: "{{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" diff --git a/upi/openstack/05_compute-nodes.yaml b/upi/openstack/05_compute-nodes.yaml index 2a3e7f7d0c4..cfacd63bd23 100644 --- a/upi/openstack/05_compute-nodes.yaml +++ b/upi/openstack/05_compute-nodes.yaml @@ -27,6 +27,20 @@ cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}" + - name: 'Create the Compute ports2' + os_port: + name: "{{ item.1 }}-{{ item.0 }}" + network: ipi-net2 + allowed_address_pairs: + - ip_address: "{{ os_subnet_range2 | next_nth_usable(7) }}" + with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" + register: ports + + - name: 'Set Compute ports tag2' + command: + cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}" + with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" + - name: 'List the Compute Trunks' command: cmd: "openstack network trunk list" @@ -50,4 +64,13 @@ userdata: "{{ lookup('file', 'worker.ign') | string }}" nics: - port-name: "{{ os_port_worker }}-{{ item.0 }}" + - port-name: "{{ os_port_worker2 }}-{{ item.0 }}" + with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" + + - name: 'Create the compute floating IP' + os_floating_ip: + state: present + nat_destination: "{{ os_network }}" + network: "{{ os_external_network }}" + server: "{{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" diff --git a/upi/openstack/common.yaml b/upi/openstack/common.yaml index ab7c1551cc9..89bc372d12f 100644 --- a/upi/openstack/common.yaml +++ b/upi/openstack/common.yaml @@ -9,14 +9,19 @@ set_fact: cluster_id_tag: "openshiftClusterID={{ infraID }}" os_network: "{{ infraID }}-network" + os_network2: "{{ infraID }}-network2" os_subnet: "{{ infraID }}-nodes" + os_subnet2: "{{ infraID }}-nodes2" os_router: "{{ infraID }}-external-router" # Port names os_port_api: "{{ infraID }}-api-port" os_port_ingress: "{{ infraID }}-ingress-port" os_port_bootstrap: "{{ infraID }}-bootstrap-port" + os_port_bootstrap2: "{{ infraID }}-bootstrap-port2" os_port_master: "{{ infraID }}-master-port" + os_port_master2: "{{ infraID }}-master-port2" os_port_worker: "{{ infraID }}-worker-port" + os_port_worker2: "{{ infraID }}-worker-port2" # Security groups names os_sg_master: "{{ infraID }}-master" os_sg_worker: "{{ infraID }}-worker" diff --git a/upi/openstack/down-03_bootstrap.yaml b/upi/openstack/down-03_bootstrap.yaml index 387860c3172..122fe9dbae2 100644 --- a/upi/openstack/down-03_bootstrap.yaml +++ b/upi/openstack/down-03_bootstrap.yaml @@ -19,3 +19,8 @@ os_port: name: "{{ os_port_bootstrap }}" state: absent + + - name: 'Remove the bootstrap server port2' + os_port: + name: "{{ os_port_bootstrap2 }}" + state: absent diff --git a/upi/openstack/down-04_control-plane.yaml b/upi/openstack/down-04_control-plane.yaml index a1ab5644f0f..54e954b7538 100644 --- a/upi/openstack/down-04_control-plane.yaml +++ b/upi/openstack/down-04_control-plane.yaml @@ -35,3 +35,9 @@ name: "{{ item.1 }}-{{ item.0 }}" state: absent with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}" + + - name: 'Remove the Control Plane ports2' + os_port: + name: "{{ item.1 }}-{{ item.0 }}" + state: absent + with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" diff --git a/upi/openstack/down-05_compute-nodes.yaml b/upi/openstack/down-05_compute-nodes.yaml index 803eeda5ee9..07261d807d7 100644 --- a/upi/openstack/down-05_compute-nodes.yaml +++ b/upi/openstack/down-05_compute-nodes.yaml @@ -35,3 +35,9 @@ name: "{{ item.1 }}-{{ item.0 }}" state: absent with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}" + + - name: 'Remove the Compute ports2' + os_port: + name: "{{ item.1 }}-{{ item.0 }}" + state: absent + with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" From 6afddb959e436b9e1c9c1f96ab385f1218604b2d Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Mon, 13 Jul 2020 15:44:03 -0700 Subject: [PATCH 02/48] Initial Commit: Adding scripts to update Ignition files --- upi/openstack/update-master.sh | 36 ++++++++ upi/openstack/update_boot.py | 149 +++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100755 upi/openstack/update-master.sh create mode 100644 upi/openstack/update_boot.py diff --git a/upi/openstack/update-master.sh b/upi/openstack/update-master.sh new file mode 100755 index 00000000000..908aeee5862 --- /dev/null +++ b/upi/openstack/update-master.sh @@ -0,0 +1,36 @@ +for index in $(seq 0 3); do + MASTER_HOSTNAME="$INFRA_ID-master-$index\n" + sudo python3 -c "import base64, json, sys; +ignition = json.load(sys.stdin); +files = ignition['storage'].get('files', []); +files.append({'path': '/etc/hostname', 'mode': 420, 'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + base64.standard_b64encode(b'$MASTER_HOSTNAME').decode().strip(), 'verification': {}}, 'filesystem': 'root'}); + +ifcfg_ens3 = 'TYPE=Ethernet\nDEVICE=ens3\nONBOOT=yes\nBOOTPROTO=dhcp\nDEFROUTE=yes\nPROXY_METHOD=none\nBROWSER_ONLY=no\nMTU=1700\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no'.encode(); + +ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip(); + +files.append({'path': '/etc/sysconfig/network-scripts/ifcfg-ens3','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens3_b64,'verification': {}},'filesystem': 'root',}); + + +ifcfg_ens4 = 'TYPE=Ethernet\nDEVICE=ens4\nONBOOT=yes\nBOOTPROTO=dhcp\nDEFROUTE=yes\nPROXY_METHOD=none\nBROWSER_ONLY=no\nMTU=1700\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no'.encode(); + +ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() + +files.append({'path': '/etc/sysconfig/network-scripts/ifcfg-ens4','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens4_b64,'verification': {}},'filesystem': 'root',}); + + +opflex_conn = 'VLAN=yes\nTYPE=Vlan\nPHYSDEV=ens4\nVLAN_ID=4093\nREORDER_HDR=yes\nGVRP=no\nMVRP=no\nPROXY_METHOD=none\nBROWSER_ONLY=no\nBOOTPROTO=dhcp\nDEFROUTE=no\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no\nNAME=opflex-conn\nDEVICE=ens4.4093\nONBOOT=yes\nMTU=1600'.encode(); + +opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip(); + +files.append({'path': '/etc/sysconfig/network-scripts/ifcfg-opflex-conn','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + opflex_conn_b64,'verification': {}},'filesystem': 'root',}); + +route_opflex_conn = 'ADDRESS0=224.0.0.0\nNETMASK0=240.0.0.0\nMETRIC0=1000'.encode(); + +route_opflex_conn_b64 = base64.standard_b64encode(route_opflex_conn).decode().strip(); + +files.append({'path': '/etc/sysconfig/network-scripts/route-opflex-conn','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + route_opflex_conn_b64,'verification': {}},'filesystem': 'root',}) + +ignition['storage']['files'] = files; +json.dump(ignition, sys.stdout)" "$INFRA_ID-master-$index-ignition.json" +done diff --git a/upi/openstack/update_boot.py b/upi/openstack/update_boot.py new file mode 100644 index 00000000000..eae8f300444 --- /dev/null +++ b/upi/openstack/update_boot.py @@ -0,0 +1,149 @@ +import base64 +import json +import os + +with open('bootstrap.ign', 'r') as f: + ignition = json.load(f) + +files = ignition['storage'].get('files', []) + +infra_id = os.environ.get('INFRA_ID', 'openshift').encode() +hostname_b64 = base64.standard_b64encode(infra_id + b'-bootstrap\n').decode().strip() +files.append( +{ + 'path': '/etc/hostname', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + hostname_b64, + 'verification': {} + }, + 'filesystem': 'root', +}) + +dhcp_client_conf_b64 = base64.standard_b64encode(b'[main]\ndhcp=dhclient\n').decode().strip() +files.append( +{ + 'path': '/etc/NetworkManager/conf.d/dhcp-client.conf', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + dhcp_client_conf_b64, + 'verification': {} + }, + 'filesystem': 'root', +}) + +dhclient_cont_b64 = base64.standard_b64encode(b'send dhcp-client-identifier = hardware;\nprepend domain-name-servers 127.0.0.1;\n').decode().strip() +files.append( +{ + 'path': '/etc/dhcp/dhclient.conf', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + dhclient_cont_b64, + 'verification': {} + }, + 'filesystem': 'root' +}) + +ifcfg_ens3 = """TYPE=Ethernet +DEVICE=ens3 +ONBOOT=yes +BOOTPROTO=dhcp +DEFROUTE=yes +PROXY_METHOD=none +BROWSER_ONLY=no +MTU=1700 +IPV4_FAILURE_FATAL=no +IPV6INIT=no""".encode() + +ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip() + +files.append( +{ + 'path': '/etc/sysconfig/network-scripts/ifcfg-ens3', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens3_b64, + 'verification': {} + }, + 'filesystem': 'root', +}) + +ifcfg_ens4 = """TYPE=Ethernet +DEVICE=ens4 +ONBOOT=yes +BOOTPROTO=dhcp +DEFROUTE=no +PROXY_METHOD=none +BROWSER_ONLY=no +MTU=1700 +IPV4_FAILURE_FATAL=no +IPV6INIT=no""".encode() + +ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() + +files.append( +{ + 'path': '/etc/sysconfig/network-scripts/ifcfg-ens4', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens4_b64, + 'verification': {} + }, + 'filesystem': 'root', +}) + + +opflex_conn = """VLAN=yes +TYPE=Vlan +PHYSDEV=ens4 +VLAN_ID=4093 +REORDER_HDR=yes +GVRP=no +MVRP=no +PROXY_METHOD=none +BROWSER_ONLY=no +BOOTPROTO=dhcp +DEFROUTE=no +IPV4_FAILURE_FATAL=no +IPV6INIT=no +NAME=opflex-conn +DEVICE=ens4.4093 +ONBOOT=yes +MTU=1600""".encode() + +opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip() + +files.append( +{ + 'path': '/etc/sysconfig/network-scripts/ifcfg-opflex-conn', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + opflex_conn_b64, + 'verification': {} + }, + 'filesystem': 'root', +}) + +route_opflex_conn = """ADDRESS0=224.0.0.0 +NETMASK0=240.0.0.0 +METRIC0=1000""".encode() + +route_opflex_conn_b64 = base64.standard_b64encode(route_opflex_conn).decode().strip() + +files.append( +{ + 'path': '/etc/sysconfig/network-scripts/route-opflex-conn', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + route_opflex_conn_b64, + 'verification': {} + }, + 'filesystem': 'root', +}) + + +ignition['storage']['files'] = files; + +with open('bootstrap.ign', 'w') as f: + json.dump(ignition, f) + From b451e523eca9a2b9c37d89ad11233b16d5e6bbaf Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Mon, 13 Jul 2020 16:12:59 -0700 Subject: [PATCH 03/48] Make ACI tasks conditional --- upi/openstack/021_network.yaml | 5 +++++ upi/openstack/03_bootstrap.yaml | 16 +++++++++++++++- upi/openstack/04_control-plane.yaml | 20 ++++++++++++++++++++ upi/openstack/05_compute-nodes.yaml | 15 +++++++++++++++ upi/openstack/down-03_bootstrap.yaml | 1 + upi/openstack/down-04_control-plane.yaml | 1 + upi/openstack/down-05_compute-nodes.yaml | 1 + upi/openstack/inventory.yaml | 2 +- 8 files changed, 59 insertions(+), 2 deletions(-) diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml index 8f03659c319..a068c6d72c7 100644 --- a/upi/openstack/021_network.yaml +++ b/upi/openstack/021_network.yaml @@ -7,10 +7,12 @@ - name: 'Create second node network' command: cmd: "neutron net-create {{ os_network2 }} --apic:nested-domain-name openshift-domain --apic:nested-domain-type openshift --apic:nested_domain_infra_vlan 4093 --apic:nested_domain_service_vlan 2003" + when: os_networking_type == "CiscoACI" - name: 'Set the second cluster network tag' command: cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_network2 }}" + when: os_networking_type == "CiscoACI" - name: 'Create a subnet2' os_subnet: @@ -20,11 +22,14 @@ cidr: "{{ os_subnet_range2 }}" allocation_pool_start: "{{ os_subnet_range2 | next_nth_usable(10) }}" allocation_pool_end: "{{ os_subnet_range2 | ipaddr('last_usable') }}" + when: os_networking_type == "CiscoACI" - name: 'Set the cluster subnet tag' command: cmd: "openstack subnet set --tag {{ cluster_id_tag }} {{ os_subnet2 }}" + when: os_networking_type == "CiscoACI" - name: 'Set dns nameserver' command: cmd: "openstack subnet set --dns-nameserver {{ dns_ip }} {{ os_subnet }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/03_bootstrap.yaml b/upi/openstack/03_bootstrap.yaml index 7fefaa4031a..cbc4260ecb4 100644 --- a/upi/openstack/03_bootstrap.yaml +++ b/upi/openstack/03_bootstrap.yaml @@ -25,17 +25,30 @@ command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap }}" - - name: 'Create the bootstrap server port' + - name: 'Create the bootstrap server port2' os_port: name: "{{ os_port_bootstrap2 }}" network: "{{ os_network2 }}" allowed_address_pairs: - ip_address: "{{ os_subnet_range | next_nth_usable(5) }}" - ip_address: "{{ os_subnet_range | next_nth_usable(6) }}" + when: os_networking_type == "CiscoACI" - name: 'Set bootstrap port tag 2' command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap2 }}" + when: os_networking_type == "CiscoACI" + + - name: 'Create the bootstrap server' + os_server: + name: "{{ os_bootstrap_server_name }}" + image: "{{ os_image_rhcos }}" + flavor: "{{ os_flavor_master }}" + userdata: "{{ lookup('file', os_bootstrap_ignition) | string }}" + auto_ip: no + nics: + - port-name: "{{ os_port_bootstrap }}" + when: os_networking_type != "CiscoACI" - name: 'Create the bootstrap server' os_server: @@ -47,6 +60,7 @@ nics: - port-name: "{{ os_port_bootstrap }}" - port-name: "{{ os_port_bootstrap2 }}" + when: os_networking_type == "CiscoACI" - name: 'Create the bootstrap floating IP' os_floating_ip: diff --git a/upi/openstack/04_control-plane.yaml b/upi/openstack/04_control-plane.yaml index 755fa1e3835..b37da22d3d3 100644 --- a/upi/openstack/04_control-plane.yaml +++ b/upi/openstack/04_control-plane.yaml @@ -39,11 +39,13 @@ - ip_address: "{{ os_subnet_range2 | next_nth_usable(7) }}" with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" register: ports + when: os_networking_type == "CiscoACI" - name: 'Set Control Plane ports tag2' command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" + when: os_networking_type == "CiscoACI" - name: 'List the Control Plane Trunks' command: @@ -74,6 +76,23 @@ - port-name: "{{ os_port_master }}-{{ item.0 }}" - port-name: "{{ os_port_master2 }}-{{ item.0 }}" with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" + when: os_networking_type == "CiscoACI" + + - name: 'Create the Control Plane servers' + os_server: + name: "{{ item.1 }}-{{ item.0 }}" + image: "{{ os_image_rhcos }}" + flavor: "{{ os_flavor_master }}" + auto_ip: no + # The ignition filename will be concatenated with the Control Plane node + # name and its 0-indexed serial number. + # In this case, the first node will look for this filename: + # "{{ infraID }}-master-0-ignition.json" + userdata: "{{ lookup('file', [item.1, item.0, 'ignition.json'] | join('-')) | string }}" + nics: + - port-name: "{{ os_port_master }}-{{ item.0 }}" + with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" + when: os_networking_type != "CiscoACI" - name: 'Create the master floating IP' os_floating_ip: @@ -82,3 +101,4 @@ network: "{{ os_external_network }}" server: "{{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/05_compute-nodes.yaml b/upi/openstack/05_compute-nodes.yaml index cfacd63bd23..ed210cbc96e 100644 --- a/upi/openstack/05_compute-nodes.yaml +++ b/upi/openstack/05_compute-nodes.yaml @@ -35,11 +35,13 @@ - ip_address: "{{ os_subnet_range2 | next_nth_usable(7) }}" with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" register: ports + when: os_networking_type == "CiscoACI" - name: 'Set Compute ports tag2' command: cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" + when: os_networking_type == "CiscoACI" - name: 'List the Compute Trunks' command: @@ -66,6 +68,18 @@ - port-name: "{{ os_port_worker }}-{{ item.0 }}" - port-name: "{{ os_port_worker2 }}-{{ item.0 }}" with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" + when: os_networking_type == "CiscoACI" + + - name: 'Create the Compute servers' + os_server: + name: "{{ item.1 }}-{{ item.0 }}" + image: "{{ os_image_rhcos }}" + flavor: "{{ os_flavor_worker }}" + auto_ip: no + userdata: "{{ lookup('file', 'worker.ign') | string }}" + nics: + - port-name: "{{ os_port_worker }}-{{ item.0 }}" + with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" - name: 'Create the compute floating IP' os_floating_ip: @@ -74,3 +88,4 @@ network: "{{ os_external_network }}" server: "{{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/down-03_bootstrap.yaml b/upi/openstack/down-03_bootstrap.yaml index 122fe9dbae2..12a227c5b62 100644 --- a/upi/openstack/down-03_bootstrap.yaml +++ b/upi/openstack/down-03_bootstrap.yaml @@ -24,3 +24,4 @@ os_port: name: "{{ os_port_bootstrap2 }}" state: absent + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/down-04_control-plane.yaml b/upi/openstack/down-04_control-plane.yaml index 54e954b7538..57d1633180c 100644 --- a/upi/openstack/down-04_control-plane.yaml +++ b/upi/openstack/down-04_control-plane.yaml @@ -41,3 +41,4 @@ name: "{{ item.1 }}-{{ item.0 }}" state: absent with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/down-05_compute-nodes.yaml b/upi/openstack/down-05_compute-nodes.yaml index 07261d807d7..8f02a6fcade 100644 --- a/upi/openstack/down-05_compute-nodes.yaml +++ b/upi/openstack/down-05_compute-nodes.yaml @@ -41,3 +41,4 @@ name: "{{ item.1 }}-{{ item.0 }}" state: absent with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index 7755d94f0fc..3459c70167d 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -23,7 +23,7 @@ all: host_prefix: '23' # Name of the SDN. # Possible values are OpenshiftSDN or Kuryr. - os_networking_type: 'OpenshiftSDN' + os_networking_type: 'CiscoACI' # Number of provisioned Control Plane nodes # 3 is the minimum number for a fully-functional cluster. From 5120a8e9739386602225be2369d703d3fe95c671 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Mon, 13 Jul 2020 18:04:01 -0700 Subject: [PATCH 04/48] Seperate floating IP playbooks for non-bootstrap nodes --- upi/openstack/041_control-plane.yaml | 14 ++++++++++++++ upi/openstack/04_control-plane.yaml | 9 --------- upi/openstack/051_compute-nodes.yaml | 14 ++++++++++++++ upi/openstack/05_compute-nodes.yaml | 10 +--------- 4 files changed, 29 insertions(+), 18 deletions(-) create mode 100644 upi/openstack/041_control-plane.yaml create mode 100644 upi/openstack/051_compute-nodes.yaml diff --git a/upi/openstack/041_control-plane.yaml b/upi/openstack/041_control-plane.yaml new file mode 100644 index 00000000000..7345a943be5 --- /dev/null +++ b/upi/openstack/041_control-plane.yaml @@ -0,0 +1,14 @@ +- import_playbook: common.yaml + +- hosts: all + gather_facts: no + + tasks: + - name: 'Create the master floating IP' + os_floating_ip: + state: present + nat_destination: "{{ os_network }}" + network: "{{ os_external_network }}" + server: "{{ item.1 }}-{{ item.0 }}" + with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/04_control-plane.yaml b/upi/openstack/04_control-plane.yaml index b37da22d3d3..26c24518769 100644 --- a/upi/openstack/04_control-plane.yaml +++ b/upi/openstack/04_control-plane.yaml @@ -93,12 +93,3 @@ - port-name: "{{ os_port_master }}-{{ item.0 }}" with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" when: os_networking_type != "CiscoACI" - - - name: 'Create the master floating IP' - os_floating_ip: - state: present - nat_destination: "{{ os_network }}" - network: "{{ os_external_network }}" - server: "{{ item.1 }}-{{ item.0 }}" - with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" - when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/051_compute-nodes.yaml b/upi/openstack/051_compute-nodes.yaml new file mode 100644 index 00000000000..039a2bcc910 --- /dev/null +++ b/upi/openstack/051_compute-nodes.yaml @@ -0,0 +1,14 @@ +- import_playbook: common.yaml + +- hosts: all + gather_facts: no + + tasks: + - name: 'Create the compute floating IP' + os_floating_ip: + state: present + nat_destination: "{{ os_network }}" + network: "{{ os_external_network }}" + server: "{{ item.1 }}-{{ item.0 }}" + with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/05_compute-nodes.yaml b/upi/openstack/05_compute-nodes.yaml index ed210cbc96e..bbd2297bcdb 100644 --- a/upi/openstack/05_compute-nodes.yaml +++ b/upi/openstack/05_compute-nodes.yaml @@ -80,12 +80,4 @@ nics: - port-name: "{{ os_port_worker }}-{{ item.0 }}" with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" - - - name: 'Create the compute floating IP' - os_floating_ip: - state: present - nat_destination: "{{ os_network }}" - network: "{{ os_external_network }}" - server: "{{ item.1 }}-{{ item.0 }}" - with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" - when: os_networking_type == "CiscoACI" + when: os_networking_type != "CiscoACI" From 75add5c0f60f23a4c506d8be31b541c80b6d5bff Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Tue, 14 Jul 2020 11:35:52 -0700 Subject: [PATCH 05/48] Separate playbook for CiscoACI variables and minor changes --- upi/openstack/03_bootstrap.yaml | 4 ++-- upi/openstack/04_control-plane.yaml | 4 ++-- upi/openstack/05_compute-nodes.yaml | 4 ++-- upi/openstack/common.yaml | 9 ++++----- upi/openstack/common_ciscoaci.yaml | 14 ++++++++++++++ upi/openstack/down-03_bootstrap.yaml | 2 +- upi/openstack/down-04_control-plane.yaml | 2 +- upi/openstack/down-05_compute-nodes.yaml | 2 +- 8 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 upi/openstack/common_ciscoaci.yaml diff --git a/upi/openstack/03_bootstrap.yaml b/upi/openstack/03_bootstrap.yaml index cbc4260ecb4..e7b5be38b12 100644 --- a/upi/openstack/03_bootstrap.yaml +++ b/upi/openstack/03_bootstrap.yaml @@ -25,7 +25,7 @@ command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap }}" - - name: 'Create the bootstrap server port2' + - name: 'Create the second bootstrap server port' os_port: name: "{{ os_port_bootstrap2 }}" network: "{{ os_network2 }}" @@ -34,7 +34,7 @@ - ip_address: "{{ os_subnet_range | next_nth_usable(6) }}" when: os_networking_type == "CiscoACI" - - name: 'Set bootstrap port tag 2' + - name: 'Set second bootstrap port tag' command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap2 }}" when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/04_control-plane.yaml b/upi/openstack/04_control-plane.yaml index 26c24518769..2e7531cec33 100644 --- a/upi/openstack/04_control-plane.yaml +++ b/upi/openstack/04_control-plane.yaml @@ -29,7 +29,7 @@ cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}" - - name: 'Create the Control Plane ports 2' + - name: 'Create the second Control Plane ports' os_port: name: "{{ item.1 }}-{{ item.0 }}" network: "{{ os_network2 }}" @@ -41,7 +41,7 @@ register: ports when: os_networking_type == "CiscoACI" - - name: 'Set Control Plane ports tag2' + - name: 'Set second Control Plane ports tag' command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" diff --git a/upi/openstack/05_compute-nodes.yaml b/upi/openstack/05_compute-nodes.yaml index bbd2297bcdb..a6345024e3b 100644 --- a/upi/openstack/05_compute-nodes.yaml +++ b/upi/openstack/05_compute-nodes.yaml @@ -27,7 +27,7 @@ cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}" - - name: 'Create the Compute ports2' + - name: 'Create the second Compute ports' os_port: name: "{{ item.1 }}-{{ item.0 }}" network: ipi-net2 @@ -37,7 +37,7 @@ register: ports when: os_networking_type == "CiscoACI" - - name: 'Set Compute ports tag2' + - name: 'Set second Compute ports tag' command: cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" diff --git a/upi/openstack/common.yaml b/upi/openstack/common.yaml index 89bc372d12f..4e422b7d114 100644 --- a/upi/openstack/common.yaml +++ b/upi/openstack/common.yaml @@ -9,19 +9,14 @@ set_fact: cluster_id_tag: "openshiftClusterID={{ infraID }}" os_network: "{{ infraID }}-network" - os_network2: "{{ infraID }}-network2" os_subnet: "{{ infraID }}-nodes" - os_subnet2: "{{ infraID }}-nodes2" os_router: "{{ infraID }}-external-router" # Port names os_port_api: "{{ infraID }}-api-port" os_port_ingress: "{{ infraID }}-ingress-port" os_port_bootstrap: "{{ infraID }}-bootstrap-port" - os_port_bootstrap2: "{{ infraID }}-bootstrap-port2" os_port_master: "{{ infraID }}-master-port" - os_port_master2: "{{ infraID }}-master-port2" os_port_worker: "{{ infraID }}-worker-port" - os_port_worker2: "{{ infraID }}-worker-port2" # Security groups names os_sg_master: "{{ infraID }}-master" os_sg_worker: "{{ infraID }}-worker" @@ -40,3 +35,7 @@ os_svc_subnet: "{{ infraID }}-kuryr-service-subnet" # Ignition files os_bootstrap_ignition: "{{ infraID }}-bootstrap-ignition.json" + + - name: 'Import variables for CiscoACI CNI' + import_playbook: common_ciscoaci.yaml + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/common_ciscoaci.yaml b/upi/openstack/common_ciscoaci.yaml new file mode 100644 index 00000000000..3939955ebaa --- /dev/null +++ b/upi/openstack/common_ciscoaci.yaml @@ -0,0 +1,14 @@ +- hosts: localhost + gather_facts: no + + vars_files: + - metadata.json + + tasks: + - name: 'Compute CiscoACI resource names' + set_fact: + os_network2: "{{ infraID }}-network2" + os_subnet2: "{{ infraID }}-nodes2" + os_port_bootstrap2: "{{ infraID }}-bootstrap-port2" + os_port_master2: "{{ infraID }}-master-port2" + os_port_worker2: "{{ infraID }}-worker-port2 diff --git a/upi/openstack/down-03_bootstrap.yaml b/upi/openstack/down-03_bootstrap.yaml index 12a227c5b62..959291ff193 100644 --- a/upi/openstack/down-03_bootstrap.yaml +++ b/upi/openstack/down-03_bootstrap.yaml @@ -20,7 +20,7 @@ name: "{{ os_port_bootstrap }}" state: absent - - name: 'Remove the bootstrap server port2' + - name: 'Remove the second bootstrap server port' os_port: name: "{{ os_port_bootstrap2 }}" state: absent diff --git a/upi/openstack/down-04_control-plane.yaml b/upi/openstack/down-04_control-plane.yaml index 57d1633180c..5b2c0baacfa 100644 --- a/upi/openstack/down-04_control-plane.yaml +++ b/upi/openstack/down-04_control-plane.yaml @@ -36,7 +36,7 @@ state: absent with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}" - - name: 'Remove the Control Plane ports2' + - name: 'Remove the second Control Plane ports' os_port: name: "{{ item.1 }}-{{ item.0 }}" state: absent diff --git a/upi/openstack/down-05_compute-nodes.yaml b/upi/openstack/down-05_compute-nodes.yaml index 8f02a6fcade..54eb09bda16 100644 --- a/upi/openstack/down-05_compute-nodes.yaml +++ b/upi/openstack/down-05_compute-nodes.yaml @@ -36,7 +36,7 @@ state: absent with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}" - - name: 'Remove the Compute ports2' + - name: 'Remove the second Compute ports' os_port: name: "{{ item.1 }}-{{ item.0 }}" state: absent From 4a464f591f91083586d4efa0919d688a6ed85613 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Tue, 14 Jul 2020 18:28:45 -0700 Subject: [PATCH 06/48] Read CiscoACI fields from inventory and minor fixes 1. Read fields node_interface, opflex_interface, neutron_network_mtu(default 1500) and infra_vlan from inventory.yaml to create ignition files 2. Use neutron_network_mtu field from [1] to set MTU for the first neutron network 3. Run ./update_control.py to generate master ignition files --- upi/openstack/021_network.yaml | 7 ++- upi/openstack/common.yaml | 2 +- upi/openstack/common_ciscoaci.yaml | 21 +++------ upi/openstack/inventory.yaml | 23 +++++---- .../{update-master.sh => update-control.sh} | 19 +++++--- upi/openstack/update_boot.py | 47 +++++++++++++------ upi/openstack/update_control.py | 25 ++++++++++ 7 files changed, 100 insertions(+), 44 deletions(-) rename upi/openstack/{update-master.sh => update-control.sh} (67%) create mode 100644 upi/openstack/update_control.py diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml index a068c6d72c7..6cd15d27ff8 100644 --- a/upi/openstack/021_network.yaml +++ b/upi/openstack/021_network.yaml @@ -4,9 +4,14 @@ gather_facts: no tasks: + - name: 'Set MTU for the first neutron network' + command: + cmd: "openstack networrk set {{ os_network }} --mtu {{ neutron_network_mtu }}" + when: os_networking_type == "CiscoACI" + - name: 'Create second node network' command: - cmd: "neutron net-create {{ os_network2 }} --apic:nested-domain-name openshift-domain --apic:nested-domain-type openshift --apic:nested_domain_infra_vlan 4093 --apic:nested_domain_service_vlan 2003" + cmd: "neutron net-create {{ os_network2 }} --apic:nested-domain-name openshift-domain --apic:nested-domain-type openshift --apic:nested_domain_infra_vlan {{ infra_vlan }} --apic:nested_domain_service_vlan {{ service_vlan }}" when: os_networking_type == "CiscoACI" - name: 'Set the second cluster network tag' diff --git a/upi/openstack/common.yaml b/upi/openstack/common.yaml index 4e422b7d114..590b032541c 100644 --- a/upi/openstack/common.yaml +++ b/upi/openstack/common.yaml @@ -37,5 +37,5 @@ os_bootstrap_ignition: "{{ infraID }}-bootstrap-ignition.json" - name: 'Import variables for CiscoACI CNI' - import_playbook: common_ciscoaci.yaml + include: common_ciscoaci.yaml when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/common_ciscoaci.yaml b/upi/openstack/common_ciscoaci.yaml index 3939955ebaa..5e3451e4b04 100644 --- a/upi/openstack/common_ciscoaci.yaml +++ b/upi/openstack/common_ciscoaci.yaml @@ -1,14 +1,7 @@ -- hosts: localhost - gather_facts: no - - vars_files: - - metadata.json - - tasks: - - name: 'Compute CiscoACI resource names' - set_fact: - os_network2: "{{ infraID }}-network2" - os_subnet2: "{{ infraID }}-nodes2" - os_port_bootstrap2: "{{ infraID }}-bootstrap-port2" - os_port_master2: "{{ infraID }}-master-port2" - os_port_worker2: "{{ infraID }}-worker-port2 +- name: 'Compute CiscoACI resource names' + set_fact: + os_network2: "{{ infraID }}-network2" + os_subnet2: "{{ infraID }}-nodes2" + os_port_bootstrap2: "{{ infraID }}-bootstrap-port2" + os_port_master2: "{{ infraID }}-master-port2" + os_port_worker2: "{{ infraID }}-worker-port2" diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index 3459c70167d..0df45e7e00f 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -5,20 +5,22 @@ all: ansible_python_interpreter: "{{ansible_playbook_python}}" # User-provided values - os_subnet_range: '10.0.0.0/16' - os_flavor_master: 'm1.xlarge' - os_flavor_worker: 'm1.large' - os_image_rhcos: 'rhcos' - os_external_network: 'external' + os_subnet_range: '15.11.0.0/24' + os_subnet_range2: '16.11.0.0/24' + os_flavor_master: 'aci_rhel_huge' + os_flavor_worker: 'aci_rhel_huge' + os_image_rhcos: 'rhcos-4.4' + os_external_network: 'l3out-2' # OpenShift API floating IP address - os_api_fip: '203.0.113.23' + os_api_fip: '60.60.60.6' + dns_ip: '172.28.184.18' # OpenShift Ingress floating IP address - os_ingress_fip: '203.0.113.19' + os_ingress_fip: '60.60.60.8' # Service subnet cidr svc_subnet_range: '172.30.0.0/16' os_svc_network_range: '172.30.0.0/15' # Subnet pool prefixes - cluster_network_cidrs: '10.128.0.0/14' + cluster_network_cidrs: '15.128.0.0/14' # Subnet pool prefix length host_prefix: '23' # Name of the SDN. @@ -32,3 +34,8 @@ all: # Number of provisioned Compute nodes. # 3 is the minimum number for a fully-functional cluster. os_compute_nodes_number: 3 + + infra_vlan: 4093 + service_vlan: 2003 + opflex_interface: ens4 + node_interface: ens3 diff --git a/upi/openstack/update-master.sh b/upi/openstack/update-control.sh similarity index 67% rename from upi/openstack/update-master.sh rename to upi/openstack/update-control.sh index 908aeee5862..54c777b0294 100755 --- a/upi/openstack/update-master.sh +++ b/upi/openstack/update-control.sh @@ -1,25 +1,32 @@ -for index in $(seq 0 3); do +#!/bin/bash +# Script that creates control plane ignition files based on given args: + +CONTROL_COUNT=$1 +NODE_IFC=$2 +OPFLEX_IFC=$3 +INFRA_VLAN=$4 +MTU=$5 + +for index in $(seq 0 $CONTROL_COUNT); do MASTER_HOSTNAME="$INFRA_ID-master-$index\n" sudo python3 -c "import base64, json, sys; ignition = json.load(sys.stdin); files = ignition['storage'].get('files', []); files.append({'path': '/etc/hostname', 'mode': 420, 'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + base64.standard_b64encode(b'$MASTER_HOSTNAME').decode().strip(), 'verification': {}}, 'filesystem': 'root'}); -ifcfg_ens3 = 'TYPE=Ethernet\nDEVICE=ens3\nONBOOT=yes\nBOOTPROTO=dhcp\nDEFROUTE=yes\nPROXY_METHOD=none\nBROWSER_ONLY=no\nMTU=1700\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no'.encode(); +ifcfg_ens3 = 'TYPE=Ethernet\nDEVICE=$NODE_IFC\nONBOOT=yes\nBOOTPROTO=dhcp\nDEFROUTE=yes\nPROXY_METHOD=none\nBROWSER_ONLY=no\nMTU=$MTU\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no'.encode(); ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip(); files.append({'path': '/etc/sysconfig/network-scripts/ifcfg-ens3','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens3_b64,'verification': {}},'filesystem': 'root',}); - -ifcfg_ens4 = 'TYPE=Ethernet\nDEVICE=ens4\nONBOOT=yes\nBOOTPROTO=dhcp\nDEFROUTE=yes\nPROXY_METHOD=none\nBROWSER_ONLY=no\nMTU=1700\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no'.encode(); +ifcfg_ens4 = 'TYPE=Ethernet\nDEVICE=$OPFLEX_IFC\nONBOOT=yes\nBOOTPROTO=dhcp\nDEFROUTE=yes\nPROXY_METHOD=none\nBROWSER_ONLY=no\nMTU=$MTU\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no'.encode(); ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() files.append({'path': '/etc/sysconfig/network-scripts/ifcfg-ens4','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens4_b64,'verification': {}},'filesystem': 'root',}); - -opflex_conn = 'VLAN=yes\nTYPE=Vlan\nPHYSDEV=ens4\nVLAN_ID=4093\nREORDER_HDR=yes\nGVRP=no\nMVRP=no\nPROXY_METHOD=none\nBROWSER_ONLY=no\nBOOTPROTO=dhcp\nDEFROUTE=no\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no\nNAME=opflex-conn\nDEVICE=ens4.4093\nONBOOT=yes\nMTU=1600'.encode(); +opflex_conn = 'VLAN=yes\nTYPE=Vlan\nPHYSDEV=$OPFLEX_IFC\nVLAN_ID=$INFRA_VLAN\nREORDER_HDR=yes\nGVRP=no\nMVRP=no\nPROXY_METHOD=none\nBROWSER_ONLY=no\nBOOTPROTO=dhcp\nDEFROUTE=no\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no\nNAME=opflex-conn\nDEVICE=$OPFLEX_IFC.$INFRA_VLAN\nONBOOT=yes\nMTU=$MTU'.encode(); opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip(); diff --git a/upi/openstack/update_boot.py b/upi/openstack/update_boot.py index eae8f300444..96bd63c687f 100644 --- a/upi/openstack/update_boot.py +++ b/upi/openstack/update_boot.py @@ -1,12 +1,32 @@ import base64 import json import os +import yaml with open('bootstrap.ign', 'r') as f: ignition = json.load(f) files = ignition['storage'].get('files', []) +# Read inventory.yaml for CiscoACI CNI variables +with open("inventory.yaml", 'r') as stream: + try: + inventory = yaml.safe_load(stream)['all']['hosts']['localhost'] + except yaml.YAMLError as exc: + print(exc) + +if 'neutron_network_mtu' not in inventory: + neutron_network_mtu = "1500" +else: + neutron_network_mtu = str(inventory['neutron_network_mtu']) + +try: + infra_vlan = str(inventory['infra_vlan']) + node_interface = inventory['node_interface'] + opflex_interface = inventory['opflex_interface'] +except: + print("The inventory.yaml must have infra_vlan, node_interface and opflex_interface fields set") + infra_id = os.environ.get('INFRA_ID', 'openshift').encode() hostname_b64 = base64.standard_b64encode(infra_id + b'-bootstrap\n').decode().strip() files.append( @@ -44,16 +64,16 @@ 'filesystem': 'root' }) -ifcfg_ens3 = """TYPE=Ethernet -DEVICE=ens3 +ifcfg_ens3 = ("""TYPE=Ethernet +DEVICE=""" + node_interface + """ ONBOOT=yes BOOTPROTO=dhcp DEFROUTE=yes PROXY_METHOD=none BROWSER_ONLY=no -MTU=1700 +MTU=""" + neutron_network_mtu + """ IPV4_FAILURE_FATAL=no -IPV6INIT=no""".encode() +IPV6INIT=no""").encode() ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip() @@ -68,16 +88,16 @@ 'filesystem': 'root', }) -ifcfg_ens4 = """TYPE=Ethernet -DEVICE=ens4 +ifcfg_ens4 = ("""TYPE=Ethernet +DEVICE=""" + opflex_interface + """ ONBOOT=yes BOOTPROTO=dhcp DEFROUTE=no PROXY_METHOD=none BROWSER_ONLY=no -MTU=1700 +MTU=""" + neutron_network_mtu + """ IPV4_FAILURE_FATAL=no -IPV6INIT=no""".encode() +IPV6INIT=no""").encode() ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() @@ -92,11 +112,10 @@ 'filesystem': 'root', }) - -opflex_conn = """VLAN=yes +opflex_conn = ("""VLAN=yes TYPE=Vlan -PHYSDEV=ens4 -VLAN_ID=4093 +PHYSDEV=""" + opflex_interface + """ +VLAN_ID=""" + infra_vlan + """ REORDER_HDR=yes GVRP=no MVRP=no @@ -107,9 +126,9 @@ IPV4_FAILURE_FATAL=no IPV6INIT=no NAME=opflex-conn -DEVICE=ens4.4093 +DEVICE=""" + opflex_interface + """.""" + infra_vlan + """ ONBOOT=yes -MTU=1600""".encode() +MTU=""" + neutron_network_mtu).encode() opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip() diff --git a/upi/openstack/update_control.py b/upi/openstack/update_control.py new file mode 100644 index 00000000000..0eac93e6a6f --- /dev/null +++ b/upi/openstack/update_control.py @@ -0,0 +1,25 @@ +import os +import yaml + +os.system('export INFRA_ID=$(jq -r .infraID metadata.json)') +# Read inventory.yaml for CiscoACI CNI variables +with open("inventory.yaml", 'r') as stream: + try: + inventory = yaml.safe_load(stream)['all']['hosts']['localhost'] + except yaml.YAMLError as exc: + print(exc) + +if 'neutron_network_mtu' not in inventory: + neutron_network_mtu = "1500" +else: + neutron_network_mtu = str(inventory['neutron_network_mtu']) + +try: + infra_vlan = str(inventory['infra_vlan']) + node_interface = inventory['node_interface'] + opflex_interface = inventory['opflex_interface'] + os_cp_nodes_number = inventory['os_cp_nodes_number'] +except: + print("The inventory.yaml must have infra_vlan, node_interface and opflex_interface fields set") + +os.system('./update-control.sh ' + str(os_cp_nodes_number) + ' ' + node_interface + ' ' + str(opflex_interface) + ' ' + infra_vlan + ' ' + str(neutron_network_mtu)) From 12c8cb38420dd96e5f7dd10c2698ece46750dbf6 Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Thu, 16 Jul 2020 18:44:32 -0700 Subject: [PATCH 07/48] Updating update_boot.py to include changes for OCP-release-4.4 --- upi/openstack/update_boot.py | 37 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/upi/openstack/update_boot.py b/upi/openstack/update_boot.py index 96bd63c687f..41e094b2030 100644 --- a/upi/openstack/update_boot.py +++ b/upi/openstack/update_boot.py @@ -40,29 +40,22 @@ 'filesystem': 'root', }) -dhcp_client_conf_b64 = base64.standard_b64encode(b'[main]\ndhcp=dhclient\n').decode().strip() -files.append( -{ - 'path': '/etc/NetworkManager/conf.d/dhcp-client.conf', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + dhcp_client_conf_b64, - 'verification': {} +ca_cert_path = os.environ.get('OS_CACERT', '') +if ca_cert_path: + with open(ca_cert_path, 'r') as f: + ca_cert = f.read().encode() + ca_cert_b64 = base64.standard_b64encode(ca_cert).decode().strip() + + files.append( + { + 'path': '/opt/openshift/tls/cloud-ca-cert.pem', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + ca_cert_b64, + 'verification': {} }, - 'filesystem': 'root', -}) - -dhclient_cont_b64 = base64.standard_b64encode(b'send dhcp-client-identifier = hardware;\nprepend domain-name-servers 127.0.0.1;\n').decode().strip() -files.append( -{ - 'path': '/etc/dhcp/dhclient.conf', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + dhclient_cont_b64, - 'verification': {} - }, - 'filesystem': 'root' -}) + 'filesystem': 'root', + }) ifcfg_ens3 = ("""TYPE=Ethernet DEVICE=""" + node_interface + """ From 1fa6b29fe17c12be591efb2b0c8f4b2a144cafed Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Thu, 23 Jul 2020 12:28:45 -0700 Subject: [PATCH 08/48] Allowed address pairs not required for second port --- upi/openstack/03_bootstrap.yaml | 3 --- upi/openstack/04_control-plane.yaml | 4 ---- upi/openstack/05_compute-nodes.yaml | 4 +--- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/upi/openstack/03_bootstrap.yaml b/upi/openstack/03_bootstrap.yaml index e7b5be38b12..6d11940a1e2 100644 --- a/upi/openstack/03_bootstrap.yaml +++ b/upi/openstack/03_bootstrap.yaml @@ -29,9 +29,6 @@ os_port: name: "{{ os_port_bootstrap2 }}" network: "{{ os_network2 }}" - allowed_address_pairs: - - ip_address: "{{ os_subnet_range | next_nth_usable(5) }}" - - ip_address: "{{ os_subnet_range | next_nth_usable(6) }}" when: os_networking_type == "CiscoACI" - name: 'Set second bootstrap port tag' diff --git a/upi/openstack/04_control-plane.yaml b/upi/openstack/04_control-plane.yaml index 2e7531cec33..6951c6affb9 100644 --- a/upi/openstack/04_control-plane.yaml +++ b/upi/openstack/04_control-plane.yaml @@ -33,10 +33,6 @@ os_port: name: "{{ item.1 }}-{{ item.0 }}" network: "{{ os_network2 }}" - allowed_address_pairs: - - ip_address: "{{ os_subnet_range2 | next_nth_usable(5) }}" - - ip_address: "{{ os_subnet_range2 | next_nth_usable(6) }}" - - ip_address: "{{ os_subnet_range2 | next_nth_usable(7) }}" with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" register: ports when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/05_compute-nodes.yaml b/upi/openstack/05_compute-nodes.yaml index a6345024e3b..6b84f668c3e 100644 --- a/upi/openstack/05_compute-nodes.yaml +++ b/upi/openstack/05_compute-nodes.yaml @@ -30,9 +30,7 @@ - name: 'Create the second Compute ports' os_port: name: "{{ item.1 }}-{{ item.0 }}" - network: ipi-net2 - allowed_address_pairs: - - ip_address: "{{ os_subnet_range2 | next_nth_usable(7) }}" + network: "{{ os_network2 }}" with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" register: ports when: os_networking_type == "CiscoACI" From 300ae1610a3e93d4baceb663afa4d6702ecd8a2e Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Thu, 16 Jul 2020 14:57:47 -0700 Subject: [PATCH 09/48] Changes to user input, SGs, field names 1. Read VLAN values from accprovision Tar file Add field 'acc_provision_tar' to inventory.yaml like this: all: hosts: localhost: acc_provision_tar: /fullpath/to/tar/file 2. Modify Cisco ACI resource names to include prefix "aci-containers" 3. Add pod network security groups for pod network for DNS(workers) and ETCD(masters) 4. Neutron address-scope and pool creation Mandatory fields required: aci_containers_addr_scope: addr-scp-openshiftupi aci_containers_l3out_vrf: uni/tn-common/ctx-sauto_l3out-1_vrf aci_containers_subnet_pool: subpool-openshiftupi aci_containers_prefix_length: 27 --- upi/openstack/01_security-groups.yaml | 17 ++++++++ upi/openstack/021_network.yaml | 24 +++++------ upi/openstack/02_network.yaml | 16 ++++--- upi/openstack/03_bootstrap.yaml | 12 +++--- upi/openstack/04_control-plane.yaml | 12 +++--- upi/openstack/05_compute-nodes.yaml | 12 +++--- upi/openstack/common_ciscoaci.yaml | 10 ++--- upi/openstack/down-03_bootstrap.yaml | 2 +- upi/openstack/down-04_control-plane.yaml | 4 +- upi/openstack/down-05_compute-nodes.yaml | 4 +- upi/openstack/inventory.yaml | 7 ++- upi/openstack/update_boot.py | 54 +++++++++++++++++++++--- 12 files changed, 120 insertions(+), 54 deletions(-) diff --git a/upi/openstack/01_security-groups.yaml b/upi/openstack/01_security-groups.yaml index 532bac9ed7e..1f95af818a5 100644 --- a/upi/openstack/01_security-groups.yaml +++ b/upi/openstack/01_security-groups.yaml @@ -149,6 +149,14 @@ port_range_min: 2379 port_range_max: 2380 + - name: 'Create master-sg rule "etcd" for pod network' + os_security_group_rule: + security_group: "{{ os_sg_master }}" + protocol: tcp + remote_ip_prefix: "{{ cluster_network_cidrs }}" + port_range_min: 2379 + port_range_max: 2380 + - name: 'Create master-sg rule "master ingress services (TCP)"' os_security_group_rule: security_group: "{{ os_sg_master }}" @@ -192,6 +200,15 @@ port_range_min: 5353 port_range_max: 5353 + - name: 'Create worker-sg rule "DNS (UDP)"' + os_security_group_rule: + security_group: "{{ os_sg_worker }}" + remote_ip_prefix: "{{ cluster_network_cidrs }}" + protocol: udp + port_range_min: 53 + port_range_max: 53 + when: os_networking_type == "CiscoACI" + - name: 'Create worker-sg rule "Ingress HTTP"' os_security_group_rule: security_group: "{{ os_sg_worker }}" diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml index 6cd15d27ff8..a148c62a823 100644 --- a/upi/openstack/021_network.yaml +++ b/upi/openstack/021_network.yaml @@ -9,29 +9,29 @@ cmd: "openstack networrk set {{ os_network }} --mtu {{ neutron_network_mtu }}" when: os_networking_type == "CiscoACI" - - name: 'Create second node network' + - name: 'Create ACI containers node network' command: - cmd: "neutron net-create {{ os_network2 }} --apic:nested-domain-name openshift-domain --apic:nested-domain-type openshift --apic:nested_domain_infra_vlan {{ infra_vlan }} --apic:nested_domain_service_vlan {{ service_vlan }}" + cmd: "neutron net-create {{ os_aci_containers_network }} --apic:nested-domain-name openshift-domain --apic:nested-domain-type openshift --apic:nested_domain_infra_vlan {{ infra_vlan }} --apic:nested_domain_service_vlan {{ service_vlan }}" when: os_networking_type == "CiscoACI" - - name: 'Set the second cluster network tag' + - name: 'Set the ACI containers cluster network tag' command: - cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_network2 }}" + cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_aci_containers_network }}" when: os_networking_type == "CiscoACI" - - name: 'Create a subnet2' + - name: 'Create the ACI containers subnet' os_subnet: - name: "{{ os_subnet2 }}" - network_name: "{{ os_network2 }}" + name: "{{ os_aci_containers_subnet }}" + network_name: "{{ os_aci_containers_network }}" no_gateway_ip: yes - cidr: "{{ os_subnet_range2 }}" - allocation_pool_start: "{{ os_subnet_range2 | next_nth_usable(10) }}" - allocation_pool_end: "{{ os_subnet_range2 | ipaddr('last_usable') }}" + cidr: "{{ os_aci_containers_subnet_range }}" + allocation_pool_start: "{{ os_aci_containers_subnet_range | next_nth_usable(10) }}" + allocation_pool_end: "{{ os_aci_containers_subnet_range | ipaddr('last_usable') }}" when: os_networking_type == "CiscoACI" - - name: 'Set the cluster subnet tag' + - name: 'Set the ACI containers cluster subnet tag' command: - cmd: "openstack subnet set --tag {{ cluster_id_tag }} {{ os_subnet2 }}" + cmd: "openstack subnet set --tag {{ cluster_id_tag }} {{ os_aci_containers_subnet }}" when: os_networking_type == "CiscoACI" - name: 'Set dns nameserver' diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index f92c94d5e01..55fd4650ebb 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -19,13 +19,17 @@ command: cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_network }}" + - name: 'Create the cluster address-scope' + command: + cmd: "neutron address-scope-update {{ aci_containers_addr_scope }} --apic:distinguished_names type=dict VRF={{ aci_containers_l3out_vrf }}" + + - name: 'Create the subnetpool' + command: + cmd: "neutron subnetpool-create --pool-prefix {{ os_subnet_range }} --address-scope {{ aci_containers_addr_scope }} {{ aci_containers_subnet_pool }}" + - name: 'Create a subnet' - os_subnet: - name: "{{ os_subnet }}" - network_name: "{{ os_network }}" - cidr: "{{ os_subnet_range }}" - allocation_pool_start: "{{ os_subnet_range | next_nth_usable(10) }}" - allocation_pool_end: "{{ os_subnet_range | ipaddr('last_usable') }}" + command: + cmd: "openstack subnet create --network {{ os_network }} --subnet-pool {{ aci_containers_subnet_pool }} --subnet-range {{ os_subnet_range }} --allocation-pool start={{ os_subnet_range | next_nth_usable(10) }},end={{ os_subnet_range | ipaddr('last_usable') }} --dhcp {{ os_subnet }} --prefix-length {{aci_containers_prefix_length }}" - name: 'Set the cluster subnet tag' command: diff --git a/upi/openstack/03_bootstrap.yaml b/upi/openstack/03_bootstrap.yaml index 6d11940a1e2..929bdf800de 100644 --- a/upi/openstack/03_bootstrap.yaml +++ b/upi/openstack/03_bootstrap.yaml @@ -25,15 +25,15 @@ command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap }}" - - name: 'Create the second bootstrap server port' + - name: 'Create the ACI containers bootstrap server port' os_port: - name: "{{ os_port_bootstrap2 }}" - network: "{{ os_network2 }}" + name: "{{ os_aci_containers_port_bootstrap }}" + network: "{{ os_aci_containers_network }}" when: os_networking_type == "CiscoACI" - - name: 'Set second bootstrap port tag' + - name: 'Set ACI containers bootstrap port tag' command: - cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_port_bootstrap2 }}" + cmd: "openstack port set --tag {{ cluster_id_tag }} {{ os_aci_containers_port_bootstrap }}" when: os_networking_type == "CiscoACI" - name: 'Create the bootstrap server' @@ -56,7 +56,7 @@ auto_ip: no nics: - port-name: "{{ os_port_bootstrap }}" - - port-name: "{{ os_port_bootstrap2 }}" + - port-name: "{{ os_aci_containers_port_bootstrap }}" when: os_networking_type == "CiscoACI" - name: 'Create the bootstrap floating IP' diff --git a/upi/openstack/04_control-plane.yaml b/upi/openstack/04_control-plane.yaml index 6951c6affb9..146fa8d0fe7 100644 --- a/upi/openstack/04_control-plane.yaml +++ b/upi/openstack/04_control-plane.yaml @@ -29,18 +29,18 @@ cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}" - - name: 'Create the second Control Plane ports' + - name: 'Create the ACI containers Control Plane ports' os_port: name: "{{ item.1 }}-{{ item.0 }}" - network: "{{ os_network2 }}" - with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" + network: "{{ os_aci_containers_network }}" + with_indexed_items: "{{ [os_aci_containers_port_master] * os_cp_nodes_number }}" register: ports when: os_networking_type == "CiscoACI" - - name: 'Set second Control Plane ports tag' + - name: 'Set ACI containers Control Plane ports tag' command: cmd: "openstack port set --tag {{ cluster_id_tag }} {{ item.1 }}-{{ item.0 }}" - with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" + with_indexed_items: "{{ [os_aci_containers_port_master] * os_cp_nodes_number }}" when: os_networking_type == "CiscoACI" - name: 'List the Control Plane Trunks' @@ -70,7 +70,7 @@ userdata: "{{ lookup('file', [item.1, item.0, 'ignition.json'] | join('-')) | string }}" nics: - port-name: "{{ os_port_master }}-{{ item.0 }}" - - port-name: "{{ os_port_master2 }}-{{ item.0 }}" + - port-name: "{{ os_aci_containers_port_master }}-{{ item.0 }}" with_indexed_items: "{{ [os_cp_server_name] * os_cp_nodes_number }}" when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/05_compute-nodes.yaml b/upi/openstack/05_compute-nodes.yaml index 6b84f668c3e..577ad0cdd1b 100644 --- a/upi/openstack/05_compute-nodes.yaml +++ b/upi/openstack/05_compute-nodes.yaml @@ -27,18 +27,18 @@ cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}" with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}" - - name: 'Create the second Compute ports' + - name: 'Create the ACI containers Compute ports' os_port: name: "{{ item.1 }}-{{ item.0 }}" - network: "{{ os_network2 }}" - with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" + network: "{{ os_aci_containers_network }}" + with_indexed_items: "{{ [os_aci_containers_port_worker] * os_compute_nodes_number }}" register: ports when: os_networking_type == "CiscoACI" - - name: 'Set second Compute ports tag' + - name: 'Set ACI containers Compute ports tag' command: cmd: "openstack port set --tag {{ [cluster_id_tag] }} {{ item.1 }}-{{ item.0 }}" - with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" + with_indexed_items: "{{ [os_aci_containers_port_worker] * os_compute_nodes_number }}" when: os_networking_type == "CiscoACI" - name: 'List the Compute Trunks' @@ -64,7 +64,7 @@ userdata: "{{ lookup('file', 'worker.ign') | string }}" nics: - port-name: "{{ os_port_worker }}-{{ item.0 }}" - - port-name: "{{ os_port_worker2 }}-{{ item.0 }}" + - port-name: "{{ os_aci_containers_port_worker }}-{{ item.0 }}" with_indexed_items: "{{ [os_compute_server_name] * os_compute_nodes_number }}" when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/common_ciscoaci.yaml b/upi/openstack/common_ciscoaci.yaml index 5e3451e4b04..8bb06d4f57c 100644 --- a/upi/openstack/common_ciscoaci.yaml +++ b/upi/openstack/common_ciscoaci.yaml @@ -1,7 +1,7 @@ - name: 'Compute CiscoACI resource names' set_fact: - os_network2: "{{ infraID }}-network2" - os_subnet2: "{{ infraID }}-nodes2" - os_port_bootstrap2: "{{ infraID }}-bootstrap-port2" - os_port_master2: "{{ infraID }}-master-port2" - os_port_worker2: "{{ infraID }}-worker-port2" + os_aci_containers_network: "{{ infraID }}-acicontainers-network" + os_aci_containers_subnet: "{{ infraID }}-acicontainers-nodes" + os_aci_containers_port_bootstrap: "{{ infraID }}-acicontainers-bootstrap-port" + os_aci_containers_port_master: "{{ infraID }}-acicontainers-master-port" + os_aci_containers_port_worker: "{{ infraID }}-acicontainers-worker-port" diff --git a/upi/openstack/down-03_bootstrap.yaml b/upi/openstack/down-03_bootstrap.yaml index 959291ff193..fe8c13f23a8 100644 --- a/upi/openstack/down-03_bootstrap.yaml +++ b/upi/openstack/down-03_bootstrap.yaml @@ -22,6 +22,6 @@ - name: 'Remove the second bootstrap server port' os_port: - name: "{{ os_port_bootstrap2 }}" + name: "{{ os_aci_containers_port_bootstrap }}" state: absent when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/down-04_control-plane.yaml b/upi/openstack/down-04_control-plane.yaml index 5b2c0baacfa..192d805c90b 100644 --- a/upi/openstack/down-04_control-plane.yaml +++ b/upi/openstack/down-04_control-plane.yaml @@ -36,9 +36,9 @@ state: absent with_indexed_items: "{{ [os_port_master] * os_cp_nodes_number }}" - - name: 'Remove the second Control Plane ports' + - name: 'Remove the ACI containers Control Plane ports' os_port: name: "{{ item.1 }}-{{ item.0 }}" state: absent - with_indexed_items: "{{ [os_port_master2] * os_cp_nodes_number }}" + with_indexed_items: "{{ [os_aci_containers_port_master] * os_cp_nodes_number }}" when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/down-05_compute-nodes.yaml b/upi/openstack/down-05_compute-nodes.yaml index 54eb09bda16..7ba9e2405a1 100644 --- a/upi/openstack/down-05_compute-nodes.yaml +++ b/upi/openstack/down-05_compute-nodes.yaml @@ -36,9 +36,9 @@ state: absent with_indexed_items: "{{ [os_port_worker] * os_compute_nodes_number }}" - - name: 'Remove the second Compute ports' + - name: 'Remove the ACI containers Compute ports' os_port: name: "{{ item.1 }}-{{ item.0 }}" state: absent - with_indexed_items: "{{ [os_port_worker2] * os_compute_nodes_number }}" + with_indexed_items: "{{ [os_aci_containers_port_worker] * os_compute_nodes_number }}" when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index 0df45e7e00f..babc9b61e26 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -35,7 +35,10 @@ all: # 3 is the minimum number for a fully-functional cluster. os_compute_nodes_number: 3 - infra_vlan: 4093 - service_vlan: 2003 + acc_provision_tar: /fullpath/to/tar/file opflex_interface: ens4 node_interface: ens3 + aci_containers_addr_scope: addr-scp-openshiftupi + aci_containers_l3out_vrf: uni/tn-common/ctx-sauto_l3out-1_vrf + aci_containers_subnet_pool: subpool-openshiftupi + aci_containers_prefix_length: 27 diff --git a/upi/openstack/update_boot.py b/upi/openstack/update_boot.py index 41e094b2030..b80239cabfd 100644 --- a/upi/openstack/update_boot.py +++ b/upi/openstack/update_boot.py @@ -1,6 +1,7 @@ import base64 import json import os +import tarfile import yaml with open('bootstrap.ign', 'r') as f: @@ -15,18 +16,59 @@ except yaml.YAMLError as exc: print(exc) -if 'neutron_network_mtu' not in inventory: - neutron_network_mtu = "1500" -else: - neutron_network_mtu = str(inventory['neutron_network_mtu']) +# Get accprovision tar path from inventory +try: + acc_provision_tar = inventory['acc_provision_tar'] +except: + print("inventory.yaml should have acc_provision_tar field") + +# Read acc-provision for vlan values +extract_to = './accProvisionTar' +tar = tarfile.open(acc_provision_tar, "r:gz") +tar.extractall(extract_to) +tar.close() + +data = '' +for filename in os.listdir(extract_to): + if 'ConfigMap-aci-containers-config' in filename: + filepath = "%s/%s" % (extract_to, filename) + with open(filepath, 'r') as stream: + try: + data = yaml.safe_load(stream)['data']['host-agent-config'] + except yaml.YAMLError as exc: + print(exc) + +# Extract host-agent-config and obtain vlan values +try: + json_data = json.loads(data) + aci_infra_vlan = json_data['aci-infra-vlan'] + service_vlan = json_data['service-vlan'] +except: + print("Couldn't extract host-agent-config from aci-containers ConfigMap") +# Set infra_vlan field in inventory.yaml using accprovision tar value +try: + with open("inventory.yaml", 'r') as stream: + cur_yaml = yaml.safe_load(stream) + cur_yaml['all']['hosts']['localhost']['infra_vlan'] = aci_infra_vlan + + if cur_yaml: + with open("inventory.yaml",'w') as yamlfile: + yaml.safe_dump(cur_yaml, yamlfile) +except: + print("Unable to edit inventory.yaml") try: - infra_vlan = str(inventory['infra_vlan']) node_interface = inventory['node_interface'] opflex_interface = inventory['opflex_interface'] except: - print("The inventory.yaml must have infra_vlan, node_interface and opflex_interface fields set") + print("The inventory.yaml must have node_interface and opflex_interface fields set") + +if 'neutron_network_mtu' not in inventory: + neutron_network_mtu = "1500" +else: + neutron_network_mtu = str(inventory['neutron_network_mtu']) +infra_vlan = str(aci_infra_vlan) infra_id = os.environ.get('INFRA_ID', 'openshift').encode() hostname_b64 = base64.standard_b64encode(infra_id + b'-bootstrap\n').decode().strip() files.append( From 34205b2fdc7d0af86fd53a445eeeca4715728380 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Thu, 23 Jul 2020 19:55:43 -0700 Subject: [PATCH 10/48] Missing CiscoACI condition for pod net SG --- upi/openstack/01_security-groups.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/upi/openstack/01_security-groups.yaml b/upi/openstack/01_security-groups.yaml index 1f95af818a5..ff5ad0cd642 100644 --- a/upi/openstack/01_security-groups.yaml +++ b/upi/openstack/01_security-groups.yaml @@ -156,6 +156,7 @@ remote_ip_prefix: "{{ cluster_network_cidrs }}" port_range_min: 2379 port_range_max: 2380 + when: os_networking_type == "CiscoACI" - name: 'Create master-sg rule "master ingress services (TCP)"' os_security_group_rule: From 6a344fe86b0200dbb2d2e02c1b40ca566aa8ba17 Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Fri, 24 Jul 2020 15:06:14 -0700 Subject: [PATCH 11/48] Consolidating the scripts to update ignition files into one single file --- upi/openstack/update-control.sh | 43 ------ upi/openstack/update_boot.py | 203 ---------------------------- upi/openstack/update_control.py | 25 ---- upi/openstack/update_ign.py | 228 ++++++++++++++++++++++++++++++++ 4 files changed, 228 insertions(+), 271 deletions(-) delete mode 100755 upi/openstack/update-control.sh delete mode 100644 upi/openstack/update_boot.py delete mode 100644 upi/openstack/update_control.py create mode 100644 upi/openstack/update_ign.py diff --git a/upi/openstack/update-control.sh b/upi/openstack/update-control.sh deleted file mode 100755 index 54c777b0294..00000000000 --- a/upi/openstack/update-control.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Script that creates control plane ignition files based on given args: - -CONTROL_COUNT=$1 -NODE_IFC=$2 -OPFLEX_IFC=$3 -INFRA_VLAN=$4 -MTU=$5 - -for index in $(seq 0 $CONTROL_COUNT); do - MASTER_HOSTNAME="$INFRA_ID-master-$index\n" - sudo python3 -c "import base64, json, sys; -ignition = json.load(sys.stdin); -files = ignition['storage'].get('files', []); -files.append({'path': '/etc/hostname', 'mode': 420, 'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + base64.standard_b64encode(b'$MASTER_HOSTNAME').decode().strip(), 'verification': {}}, 'filesystem': 'root'}); - -ifcfg_ens3 = 'TYPE=Ethernet\nDEVICE=$NODE_IFC\nONBOOT=yes\nBOOTPROTO=dhcp\nDEFROUTE=yes\nPROXY_METHOD=none\nBROWSER_ONLY=no\nMTU=$MTU\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no'.encode(); - -ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip(); - -files.append({'path': '/etc/sysconfig/network-scripts/ifcfg-ens3','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens3_b64,'verification': {}},'filesystem': 'root',}); - -ifcfg_ens4 = 'TYPE=Ethernet\nDEVICE=$OPFLEX_IFC\nONBOOT=yes\nBOOTPROTO=dhcp\nDEFROUTE=yes\nPROXY_METHOD=none\nBROWSER_ONLY=no\nMTU=$MTU\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no'.encode(); - -ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() - -files.append({'path': '/etc/sysconfig/network-scripts/ifcfg-ens4','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens4_b64,'verification': {}},'filesystem': 'root',}); - -opflex_conn = 'VLAN=yes\nTYPE=Vlan\nPHYSDEV=$OPFLEX_IFC\nVLAN_ID=$INFRA_VLAN\nREORDER_HDR=yes\nGVRP=no\nMVRP=no\nPROXY_METHOD=none\nBROWSER_ONLY=no\nBOOTPROTO=dhcp\nDEFROUTE=no\nIPV4_FAILURE_FATAL=no\nIPV6INIT=no\nNAME=opflex-conn\nDEVICE=$OPFLEX_IFC.$INFRA_VLAN\nONBOOT=yes\nMTU=$MTU'.encode(); - -opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip(); - -files.append({'path': '/etc/sysconfig/network-scripts/ifcfg-opflex-conn','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + opflex_conn_b64,'verification': {}},'filesystem': 'root',}); - -route_opflex_conn = 'ADDRESS0=224.0.0.0\nNETMASK0=240.0.0.0\nMETRIC0=1000'.encode(); - -route_opflex_conn_b64 = base64.standard_b64encode(route_opflex_conn).decode().strip(); - -files.append({'path': '/etc/sysconfig/network-scripts/route-opflex-conn','mode': 420,'contents': {'source': 'data:text/plain;charset=utf-8;base64,' + route_opflex_conn_b64,'verification': {}},'filesystem': 'root',}) - -ignition['storage']['files'] = files; -json.dump(ignition, sys.stdout)" "$INFRA_ID-master-$index-ignition.json" -done diff --git a/upi/openstack/update_boot.py b/upi/openstack/update_boot.py deleted file mode 100644 index b80239cabfd..00000000000 --- a/upi/openstack/update_boot.py +++ /dev/null @@ -1,203 +0,0 @@ -import base64 -import json -import os -import tarfile -import yaml - -with open('bootstrap.ign', 'r') as f: - ignition = json.load(f) - -files = ignition['storage'].get('files', []) - -# Read inventory.yaml for CiscoACI CNI variables -with open("inventory.yaml", 'r') as stream: - try: - inventory = yaml.safe_load(stream)['all']['hosts']['localhost'] - except yaml.YAMLError as exc: - print(exc) - -# Get accprovision tar path from inventory -try: - acc_provision_tar = inventory['acc_provision_tar'] -except: - print("inventory.yaml should have acc_provision_tar field") - -# Read acc-provision for vlan values -extract_to = './accProvisionTar' -tar = tarfile.open(acc_provision_tar, "r:gz") -tar.extractall(extract_to) -tar.close() - -data = '' -for filename in os.listdir(extract_to): - if 'ConfigMap-aci-containers-config' in filename: - filepath = "%s/%s" % (extract_to, filename) - with open(filepath, 'r') as stream: - try: - data = yaml.safe_load(stream)['data']['host-agent-config'] - except yaml.YAMLError as exc: - print(exc) - -# Extract host-agent-config and obtain vlan values -try: - json_data = json.loads(data) - aci_infra_vlan = json_data['aci-infra-vlan'] - service_vlan = json_data['service-vlan'] -except: - print("Couldn't extract host-agent-config from aci-containers ConfigMap") - -# Set infra_vlan field in inventory.yaml using accprovision tar value -try: - with open("inventory.yaml", 'r') as stream: - cur_yaml = yaml.safe_load(stream) - cur_yaml['all']['hosts']['localhost']['infra_vlan'] = aci_infra_vlan - - if cur_yaml: - with open("inventory.yaml",'w') as yamlfile: - yaml.safe_dump(cur_yaml, yamlfile) -except: - print("Unable to edit inventory.yaml") -try: - node_interface = inventory['node_interface'] - opflex_interface = inventory['opflex_interface'] -except: - print("The inventory.yaml must have node_interface and opflex_interface fields set") - -if 'neutron_network_mtu' not in inventory: - neutron_network_mtu = "1500" -else: - neutron_network_mtu = str(inventory['neutron_network_mtu']) - -infra_vlan = str(aci_infra_vlan) -infra_id = os.environ.get('INFRA_ID', 'openshift').encode() -hostname_b64 = base64.standard_b64encode(infra_id + b'-bootstrap\n').decode().strip() -files.append( -{ - 'path': '/etc/hostname', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + hostname_b64, - 'verification': {} - }, - 'filesystem': 'root', -}) - -ca_cert_path = os.environ.get('OS_CACERT', '') -if ca_cert_path: - with open(ca_cert_path, 'r') as f: - ca_cert = f.read().encode() - ca_cert_b64 = base64.standard_b64encode(ca_cert).decode().strip() - - files.append( - { - 'path': '/opt/openshift/tls/cloud-ca-cert.pem', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + ca_cert_b64, - 'verification': {} - }, - 'filesystem': 'root', - }) - -ifcfg_ens3 = ("""TYPE=Ethernet -DEVICE=""" + node_interface + """ -ONBOOT=yes -BOOTPROTO=dhcp -DEFROUTE=yes -PROXY_METHOD=none -BROWSER_ONLY=no -MTU=""" + neutron_network_mtu + """ -IPV4_FAILURE_FATAL=no -IPV6INIT=no""").encode() - -ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip() - -files.append( -{ - 'path': '/etc/sysconfig/network-scripts/ifcfg-ens3', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens3_b64, - 'verification': {} - }, - 'filesystem': 'root', -}) - -ifcfg_ens4 = ("""TYPE=Ethernet -DEVICE=""" + opflex_interface + """ -ONBOOT=yes -BOOTPROTO=dhcp -DEFROUTE=no -PROXY_METHOD=none -BROWSER_ONLY=no -MTU=""" + neutron_network_mtu + """ -IPV4_FAILURE_FATAL=no -IPV6INIT=no""").encode() - -ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() - -files.append( -{ - 'path': '/etc/sysconfig/network-scripts/ifcfg-ens4', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens4_b64, - 'verification': {} - }, - 'filesystem': 'root', -}) - -opflex_conn = ("""VLAN=yes -TYPE=Vlan -PHYSDEV=""" + opflex_interface + """ -VLAN_ID=""" + infra_vlan + """ -REORDER_HDR=yes -GVRP=no -MVRP=no -PROXY_METHOD=none -BROWSER_ONLY=no -BOOTPROTO=dhcp -DEFROUTE=no -IPV4_FAILURE_FATAL=no -IPV6INIT=no -NAME=opflex-conn -DEVICE=""" + opflex_interface + """.""" + infra_vlan + """ -ONBOOT=yes -MTU=""" + neutron_network_mtu).encode() - -opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip() - -files.append( -{ - 'path': '/etc/sysconfig/network-scripts/ifcfg-opflex-conn', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + opflex_conn_b64, - 'verification': {} - }, - 'filesystem': 'root', -}) - -route_opflex_conn = """ADDRESS0=224.0.0.0 -NETMASK0=240.0.0.0 -METRIC0=1000""".encode() - -route_opflex_conn_b64 = base64.standard_b64encode(route_opflex_conn).decode().strip() - -files.append( -{ - 'path': '/etc/sysconfig/network-scripts/route-opflex-conn', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + route_opflex_conn_b64, - 'verification': {} - }, - 'filesystem': 'root', -}) - - -ignition['storage']['files'] = files; - -with open('bootstrap.ign', 'w') as f: - json.dump(ignition, f) - diff --git a/upi/openstack/update_control.py b/upi/openstack/update_control.py deleted file mode 100644 index 0eac93e6a6f..00000000000 --- a/upi/openstack/update_control.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import yaml - -os.system('export INFRA_ID=$(jq -r .infraID metadata.json)') -# Read inventory.yaml for CiscoACI CNI variables -with open("inventory.yaml", 'r') as stream: - try: - inventory = yaml.safe_load(stream)['all']['hosts']['localhost'] - except yaml.YAMLError as exc: - print(exc) - -if 'neutron_network_mtu' not in inventory: - neutron_network_mtu = "1500" -else: - neutron_network_mtu = str(inventory['neutron_network_mtu']) - -try: - infra_vlan = str(inventory['infra_vlan']) - node_interface = inventory['node_interface'] - opflex_interface = inventory['opflex_interface'] - os_cp_nodes_number = inventory['os_cp_nodes_number'] -except: - print("The inventory.yaml must have infra_vlan, node_interface and opflex_interface fields set") - -os.system('./update-control.sh ' + str(os_cp_nodes_number) + ' ' + node_interface + ' ' + str(opflex_interface) + ' ' + infra_vlan + ' ' + str(neutron_network_mtu)) diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py new file mode 100644 index 00000000000..c2da978ca92 --- /dev/null +++ b/upi/openstack/update_ign.py @@ -0,0 +1,228 @@ +import base64 +import json +import os +import tarfile +import yaml + + +#The script does the following things: +#Update boostrap.ign with hostname and CA certs and with additional network-scripts. +#According to the number of the master count, create the JSON files, and add hostname/network-scripts. +#According to the number of the worker count, create the JSON files, and add hostname/network-scripts. + +# Read inventory.yaml for CiscoACI CNI variables +with open("inventory.yaml", 'r') as stream: + try: + inventory = yaml.safe_load(stream)['all']['hosts']['localhost'] + except yaml.YAMLError as exc: + print(exc) + +# Get accprovision tar path from inventory +try: + acc_provision_tar = inventory['acc_provision_tar'] +except: + print("inventory.yaml should have acc_provision_tar field") + +# Read acc-provision for vlan values +extract_to = './accProvisionTar' +tar = tarfile.open(acc_provision_tar, "r:gz") +tar.extractall(extract_to) +tar.close() + +data = '' +for filename in os.listdir(extract_to): + if 'ConfigMap-aci-containers-config' in filename: + filepath = "%s/%s" % (extract_to, filename) + with open(filepath, 'r') as stream: + try: + data = yaml.safe_load(stream)['data']['host-agent-config'] + except yaml.YAMLError as exc: + print(exc) + +# Extract host-agent-config and obtain vlan values +try: + json_data = json.loads(data) + aci_infra_vlan = json_data['aci-infra-vlan'] +except: + print("Couldn't extract host-agent-config from aci-containers ConfigMap") + +# Set infra_vlan field in inventory.yaml using accprovision tar value +try: + with open("inventory.yaml", 'r') as stream: + cur_yaml = yaml.safe_load(stream) + cur_yaml['all']['hosts']['localhost']['infra_vlan'] = aci_infra_vlan + + if cur_yaml: + with open("inventory.yaml",'w') as yamlfile: + yaml.safe_dump(cur_yaml, yamlfile) +except: + print("Unable to edit inventory.yaml") +try: + node_interface = inventory['node_interface'] + opflex_interface = inventory['opflex_interface'] + master_count = inventory['os_cp_nodes_number'] + worker_count = inventory['os_compute_nodes_number'] +except: + print("Relevant Fields are missing from inventory.yaml ") + +if 'neutron_network_mtu' not in inventory: + neutron_network_mtu = "1500" +else: + neutron_network_mtu = str(inventory['neutron_network_mtu']) + +infra_vlan = str(aci_infra_vlan) +infra_id = os.environ.get('INFRA_ID', 'openshift').encode() + +def update(hostname,ignition): + files = ignition['storage'].get('files', []) + if 'bootstrap' in hostname.decode(): + ca_cert_path = os.environ.get('OS_CACERT', '') + if ca_cert_path: + with open(ca_cert_path, 'r') as f: + ca_cert = f.read().encode() + ca_cert_b64 = base64.standard_b64encode(ca_cert).decode().strip() + + files.append( + { + 'path': '/opt/openshift/tls/cloud-ca-cert.pem', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + ca_cert_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + hostname_b64 = base64.standard_b64encode(hostname).decode().strip() + files.append( + { + 'path': '/etc/hostname', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + hostname_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + ifcfg_ens3 = ("""TYPE=Ethernet + DEVICE=""" + node_interface + """ + ONBOOT=yes + BOOTPROTO=dhcp + DEFROUTE=yes + PROXY_METHOD=none + BROWSER_ONLY=no + MTU=""" + neutron_network_mtu + """ + IPV4_FAILURE_FATAL=no + IPV6INIT=no""").encode() + + ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip() + + files.append( + { + 'path': '/etc/sysconfig/network-scripts/ifcfg-ens3', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens3_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + ifcfg_ens4 = ("""TYPE=Ethernet + DEVICE=""" + opflex_interface + """ + ONBOOT=yes + BOOTPROTO=dhcp + DEFROUTE=no + PROXY_METHOD=none + BROWSER_ONLY=no + MTU=""" + neutron_network_mtu + """ + IPV4_FAILURE_FATAL=no + IPV6INIT=no""").encode() + + ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() + + files.append( + { + 'path': '/etc/sysconfig/network-scripts/ifcfg-ens4', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens4_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + opflex_conn = ("""VLAN=yes + TYPE=Vlan + PHYSDEV=""" + opflex_interface + """ + VLAN_ID=""" + infra_vlan + """ + REORDER_HDR=yes + GVRP=no + MVRP=no + PROXY_METHOD=none + BROWSER_ONLY=no + BOOTPROTO=dhcp + DEFROUTE=no + IPV4_FAILURE_FATAL=no + IPV6INIT=no + NAME=opflex-conn + DEVICE=""" + opflex_interface + """.""" + infra_vlan + """ + ONBOOT=yes + MTU=""" + neutron_network_mtu).encode() + + opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip() + + files.append( + { + 'path': '/etc/sysconfig/network-scripts/ifcfg-opflex-conn', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + opflex_conn_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + route_opflex_conn = """ADDRESS0=224.0.0.0 + NETMASK0=240.0.0.0 + METRIC0=1000""".encode() + + route_opflex_conn_b64 = base64.standard_b64encode(route_opflex_conn).decode().strip() + + files.append( + { + 'path': '/etc/sysconfig/network-scripts/route-opflex-conn', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + route_opflex_conn_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + ignition['storage']['files'] = files + return ignition + + +with open('bootstrap.ign', 'r') as f: + ignition = json.load(f) +bootstrap_hostname = infra_id + b'-bootstrap\n' +ignition = update(bootstrap_hostname,ignition) +with open('bootstrap.ign', 'w') as f: + json.dump(ignition, f) + +for index in range(0,master_count): + master_hostname = infra_id + b'-master-' + str(index).encode() + b'\n' + with open('master.ign', 'r') as f: + ignition = json.load(f) + ignition = update(master_hostname,ignition) + with open(infra_id.decode() + '-master-' + str(index) + '-ignition.json', 'w') as f: + json.dump(ignition, f) + +for index in range(0,worker_count): + master_hostname = infra_id + b'-worker-' + str(index).encode() + b'\n' + with open('worker.ign', 'r') as f: + ignition = json.load(f) + ignition = update(master_hostname,ignition) + with open(infra_id.decode() + '-worker-' + str(index) + '-ignition.json', 'w') as f: + json.dump(ignition, f) From 0e3bce6f3aacf11fdd507cedc9813e62d3e28a9d Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Mon, 27 Jul 2020 16:35:41 -0700 Subject: [PATCH 12/48] Adding relevant change to compute nodes playbook to facilitate multiple ignition.json files --- upi/openstack/05_compute-nodes.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/upi/openstack/05_compute-nodes.yaml b/upi/openstack/05_compute-nodes.yaml index 577ad0cdd1b..da3ff50427e 100644 --- a/upi/openstack/05_compute-nodes.yaml +++ b/upi/openstack/05_compute-nodes.yaml @@ -61,7 +61,11 @@ image: "{{ os_image_rhcos }}" flavor: "{{ os_flavor_worker }}" auto_ip: no - userdata: "{{ lookup('file', 'worker.ign') | string }}" + # The ignition filename will be concatenated with the Compute node + # name and its 0-indexed serial number. + # In this case, the first node will look for this filename: + # "{{ infraID }}-worker-0-ignition.json" + userdata: "{{ lookup('file', [item.1, item.0, 'ignition.json'] | join('-')) | string }}" nics: - port-name: "{{ os_port_worker }}-{{ item.0 }}" - port-name: "{{ os_aci_containers_port_worker }}-{{ item.0 }}" From 6d508d414b03ccbb76ad344f79b39127e5524794 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Mon, 27 Jul 2020 17:46:35 -0700 Subject: [PATCH 13/48] Fix typo and missing neutron_network_mtu,service_vlan in network playbook --- upi/openstack/021_network.yaml | 2 +- upi/openstack/update_ign.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml index a148c62a823..dab3dec027c 100644 --- a/upi/openstack/021_network.yaml +++ b/upi/openstack/021_network.yaml @@ -6,7 +6,7 @@ tasks: - name: 'Set MTU for the first neutron network' command: - cmd: "openstack networrk set {{ os_network }} --mtu {{ neutron_network_mtu }}" + cmd: "openstack network set {{ os_network }} --mtu {{ neutron_network_mtu }}" when: os_networking_type == "CiscoACI" - name: 'Create ACI containers node network' diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index c2da978ca92..3f134b5c851 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -43,14 +43,23 @@ try: json_data = json.loads(data) aci_infra_vlan = json_data['aci-infra-vlan'] + service_vlan = json_data['service-vlan'] except: print("Couldn't extract host-agent-config from aci-containers ConfigMap") +if 'neutron_network_mtu' not in inventory: + neutron_network_mtu = "1500" +else: + neutron_network_mtu = str(inventory['neutron_network_mtu']) + # Set infra_vlan field in inventory.yaml using accprovision tar value try: with open("inventory.yaml", 'r') as stream: cur_yaml = yaml.safe_load(stream) cur_yaml['all']['hosts']['localhost']['infra_vlan'] = aci_infra_vlan + cur_yaml['all']['hosts']['localhost']['service-vlan'] = service_vlan + if 'neutron_network_mtu' not in inventory: + cur_yaml['all']['hosts']['localhost']['neutron_network_mtu'] = neutron_network_mtu if cur_yaml: with open("inventory.yaml",'w') as yamlfile: @@ -65,11 +74,6 @@ except: print("Relevant Fields are missing from inventory.yaml ") -if 'neutron_network_mtu' not in inventory: - neutron_network_mtu = "1500" -else: - neutron_network_mtu = str(inventory['neutron_network_mtu']) - infra_vlan = str(aci_infra_vlan) infra_id = os.environ.get('INFRA_ID', 'openshift').encode() From d4dc7e84caf3018654e579194a054dd152c1a374 Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Tue, 28 Jul 2020 00:19:06 -0700 Subject: [PATCH 14/48] Changes added to playbooks to resolve issues related 1. Updated address-scope command from update to create 2. Added setting network mtu for aci-containers network 3. Added clean-up of address scope and subnet-pool in down_network playbook --- upi/openstack/021_network.yaml | 10 +++++----- upi/openstack/02_network.yaml | 7 ++++++- upi/openstack/down-02_network.yaml | 10 ++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml index dab3dec027c..4757d7674f5 100644 --- a/upi/openstack/021_network.yaml +++ b/upi/openstack/021_network.yaml @@ -4,11 +4,6 @@ gather_facts: no tasks: - - name: 'Set MTU for the first neutron network' - command: - cmd: "openstack network set {{ os_network }} --mtu {{ neutron_network_mtu }}" - when: os_networking_type == "CiscoACI" - - name: 'Create ACI containers node network' command: cmd: "neutron net-create {{ os_aci_containers_network }} --apic:nested-domain-name openshift-domain --apic:nested-domain-type openshift --apic:nested_domain_infra_vlan {{ infra_vlan }} --apic:nested_domain_service_vlan {{ service_vlan }}" @@ -34,6 +29,11 @@ cmd: "openstack subnet set --tag {{ cluster_id_tag }} {{ os_aci_containers_subnet }}" when: os_networking_type == "CiscoACI" + - name: 'Set MTU for the ACI containers node network' + command: + cmd: "openstack network set {{ os_aci_containers_network }} --mtu {{ neutron_network_mtu }}" + when: os_networking_type == "CiscoACI" + - name: 'Set dns nameserver' command: cmd: "openstack subnet set --dns-nameserver {{ dns_ip }} {{ os_subnet }}" diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index 55fd4650ebb..d28ea7a77fe 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -21,7 +21,7 @@ - name: 'Create the cluster address-scope' command: - cmd: "neutron address-scope-update {{ aci_containers_addr_scope }} --apic:distinguished_names type=dict VRF={{ aci_containers_l3out_vrf }}" + cmd: "neutron address-scope-create {{ aci_containers_addr_scope }} 4 --apic:distinguished_names type=dict VRF={{ aci_containers_l3out_vrf }}" - name: 'Create the subnetpool' command: @@ -155,3 +155,8 @@ - name: 'Attach the Ingress floating IP to Ingress port' command: cmd: "openstack floating ip set --port {{ os_port_ingress }} {{ os_ingress_fip }}" + + - name: 'Set MTU for the first neutron network' + command: + cmd: "openstack networrk set {{ os_network }} --mtu {{ neutron_network_mtu }}" + when: os_networking_type == "CiscoACI" diff --git a/upi/openstack/down-02_network.yaml b/upi/openstack/down-02_network.yaml index 15318284c49..f15cd5f20ac 100644 --- a/upi/openstack/down-02_network.yaml +++ b/upi/openstack/down-02_network.yaml @@ -68,3 +68,13 @@ when: - os_networking_type == "Kuryr" - pods_subnet_pool.stdout != "" + + - name: 'Remove the cluster subnet pool' + command: + cmd: "openstack subnet pool delete {{ aci_containers_subnet_pool }}" + when: os_networking_type == "CiscoACI" + + - name: 'Remove the cluster address-scope' + command: + cmd: "openstack address scope delete {{ aci_containers_addr_scope }}" + when: os_networking_type == "CiscoACI" From 96ac22539c251491a263ce0c022045ae76c6ecbe Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Tue, 28 Jul 2020 22:04:19 -0700 Subject: [PATCH 15/48] Refactor to change ACI CNI inventory fields, tasks and SGs 1. Remove the Ansible task to set MTU for node network 2. Delete the accProvisiontar unpacked for preprocessing 3. Open all TCP and UDP master and worker ports from the pod subnet(cluster_network_cidrs) 4. Remove FIP association for master and worker nodes --- upi/openstack/.down-02_network.yaml.swp | Bin 0 -> 12288 bytes upi/openstack/01_security-groups.yaml | 46 +++++++++++++-------- upi/openstack/021_network.yaml | 12 +++--- upi/openstack/02_network.yaml | 23 +++++++---- upi/openstack/041_control-plane.yaml | 14 ------- upi/openstack/051_compute-nodes.yaml | 14 ------- upi/openstack/down-02_network.yaml | 4 +- upi/openstack/inventory.yaml | 22 ++++++---- upi/openstack/update_ign.py | 52 ++++++++++++++++-------- 9 files changed, 100 insertions(+), 87 deletions(-) create mode 100644 upi/openstack/.down-02_network.yaml.swp delete mode 100644 upi/openstack/041_control-plane.yaml delete mode 100644 upi/openstack/051_compute-nodes.yaml diff --git a/upi/openstack/.down-02_network.yaml.swp b/upi/openstack/.down-02_network.yaml.swp new file mode 100644 index 0000000000000000000000000000000000000000..a9b31ac43f92bd06ee88f19dff7f191c454e58db GIT binary patch literal 12288 zcmeI2ONbmr7{@Dt_)e6KpvNz}V7&D1&L$A;xI&1R;DflZxwbXkH8Yjz>6)#ona!{y zIeQlL6fZswISGMy&B=oxih76_1qtdw#iOU-zq)&NdS)1A)(4_g!;hY>uCB-L`?_Y9 zt@`?<3$N0d)zb{e6O7&cjiOKEW9*%-kHDh2E2ihfu3yy&mG-9wNYDlg)7S^==jq1=)V8t4R{0IfH&X`cmv*m zH{cC;1OIyiwtI+O$1D$*Grd@z4^5oQOMmhPya8{(8}J6a0dK$?@CLjAZ@?Sy2E2j) zpaGsRwsVBB&mO?w@&Etf-~aa>XY6}$7kmOf2DiaG;BD|G=z|_e!8vdm+&RYB2jG3M z2Qu(1`1L4bzkoa77N~*aUz*!K2C&3Z$ z<0FiH1-=9yf%m``I156s1P+6r9%k%o@GiIkENFsf!2O39`wn~yZh;TMW$*@g0Xz+! z0zW^9df+Cw4z7XM!AszGcyumrJD>jW2D|}pz#I6x4G@mqmdHXz+fQ9Vm1;Ie2u7zJQl^~*gy&_$hmf-{U) zdaaDV(j^_xaHYX=8d@}^lOIk^XTJW*ENCiQDqT^EE+`0G`2#w1FSn`Yk@5i;5$3whm$%$D|x#B@))hE{7sxBD+q zdtL$$d!2GSyfGurT8b_B;2XfcqSP?J;YG~D=vAB-bjEUhc zATtqt8H)frl!e|b^D-PYqeVBBG2TtQ%^5PV!DA$QuxJS8L|}hT+~OOpB@(h~d}_1e zQq2Ef56b+AR0sNJ!%?>J4+NkXzq4;A^%Gfc+9q^g`%R8jUq=YO#d@A87EntW_Zi#W z8Lni + # generated diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index 3f134b5c851..226571c1a8b 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -1,6 +1,7 @@ import base64 import json import os +import shutil import tarfile import yaml @@ -10,18 +11,21 @@ #According to the number of the master count, create the JSON files, and add hostname/network-scripts. #According to the number of the worker count, create the JSON files, and add hostname/network-scripts. -# Read inventory.yaml for CiscoACI CNI variables -with open("inventory.yaml", 'r') as stream: +# Read inventory.yaml for CiscoACI CNI variable +original_inventory = processed_inventory = "inventory.yaml" +with open(original_inventory, 'r') as stream: try: - inventory = yaml.safe_load(stream)['all']['hosts']['localhost'] + localhost = yaml.safe_load(stream)['all']['hosts']['localhost'] + inventory = localhost['aci_cni'] except yaml.YAMLError as exc: print(exc) # Get accprovision tar path from inventory try: acc_provision_tar = inventory['acc_provision_tar'] + os_subnet_range = localhost['os_subnet_range'] except: - print("inventory.yaml should have acc_provision_tar field") + print("inventory.yaml should have acc_provision_tar and os_subnet_range fields") # Read acc-provision for vlan values extract_to = './accProvisionTar' @@ -47,30 +51,46 @@ except: print("Couldn't extract host-agent-config from aci-containers ConfigMap") -if 'neutron_network_mtu' not in inventory: +# Delete acc_provisionTar that was untarred previously +try: + shutil.rmtree(extract_to) +except OSError as e: + print ("Error: %s - %s." % (e.filename, e.strerror)) + +if 'mtu' not in inventory['network_interfaces']['opflex']: neutron_network_mtu = "1500" else: - neutron_network_mtu = str(inventory['neutron_network_mtu']) + neutron_network_mtu = str(inventory['network_interfaces']['opflex']['mtu']) + +# Returns the mask value for a subnet, for example 10.0.0.0/24 returns 24 +def get_prefix_from_subnet(subnet): + mask = None + try: + split_subnet = subnet.split("/") + mask = split_subnet[1] + except: + print("os_subnet_range not in valid format i.e a.b.c.d/e") + return mask # Set infra_vlan field in inventory.yaml using accprovision tar value try: - with open("inventory.yaml", 'r') as stream: + with open(original_inventory, 'r') as stream: cur_yaml = yaml.safe_load(stream) - cur_yaml['all']['hosts']['localhost']['infra_vlan'] = aci_infra_vlan - cur_yaml['all']['hosts']['localhost']['service-vlan'] = service_vlan - if 'neutron_network_mtu' not in inventory: - cur_yaml['all']['hosts']['localhost']['neutron_network_mtu'] = neutron_network_mtu + cur_yaml['all']['hosts']['localhost']['aci_cni']['infra_vlan'] = aci_infra_vlan + cur_yaml['all']['hosts']['localhost']['aci_cni']['service_vlan'] = service_vlan + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['subnet_prefix_length'] = get_prefix_from_subnet(os_subnet_range) + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['mtu'] = neutron_network_mtu if cur_yaml: - with open("inventory.yaml",'w') as yamlfile: + with open(processed_inventory,'w') as yamlfile: yaml.safe_dump(cur_yaml, yamlfile) except: print("Unable to edit inventory.yaml") try: - node_interface = inventory['node_interface'] - opflex_interface = inventory['opflex_interface'] - master_count = inventory['os_cp_nodes_number'] - worker_count = inventory['os_compute_nodes_number'] + node_interface = inventory['network_interfaces']['node']['name'] + opflex_interface = inventory['network_interfaces']['opflex']['name'] + master_count = localhost['os_cp_nodes_number'] + worker_count = localhost['os_compute_nodes_number'] except: print("Relevant Fields are missing from inventory.yaml ") From 3492da612700aad0d6309b4801960e63e620e68b Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Fri, 31 Jul 2020 15:41:48 -0700 Subject: [PATCH 16/48] Post-install step to create cluster SNAT policy Add snat IP and kubeconfig path to inventory inside aci_cni field like aci_cni: cluster_snat_policy_ip: kubeconfig: --- upi/openstack/inventory.yaml | 1 + upi/openstack/post-install.yaml | 14 ++++++++++++++ .../templates/cluster_snat_policy.conf.j2 | 7 +++++++ 3 files changed, 22 insertions(+) create mode 100644 upi/openstack/post-install.yaml create mode 100644 upi/openstack/templates/cluster_snat_policy.conf.j2 diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index 31551b20273..df33d37abeb 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -34,6 +34,7 @@ all: # 3 is the minimum number for a fully-functional cluster. os_compute_nodes_number: 3 aci_cni: + kubeconfig: /path/to/kubeconfig acc_provision_tar: /home/noiro/openupi/aci_deployment.yaml.tar.gz network_interfaces: node: diff --git a/upi/openstack/post-install.yaml b/upi/openstack/post-install.yaml new file mode 100644 index 00000000000..e3895756674 --- /dev/null +++ b/upi/openstack/post-install.yaml @@ -0,0 +1,14 @@ +- hosts: all + gather_facts: no + + tasks: + - name: 'Create snat policy file' + template: + src: cluster_snat_policy.conf.j2 + dest: cluster_snatpolicy.yaml + + - name: 'Create cluster SNAT policy' + k8s: + state: present + kubeconfig: "{{ aci_cni['kubeconfig'] }}" + src: cluster_snatpolicy.yaml diff --git a/upi/openstack/templates/cluster_snat_policy.conf.j2 b/upi/openstack/templates/cluster_snat_policy.conf.j2 new file mode 100644 index 00000000000..a77a8f32e03 --- /dev/null +++ b/upi/openstack/templates/cluster_snat_policy.conf.j2 @@ -0,0 +1,7 @@ +apiVersion: aci.snat/v1 +kind: SnatPolicy +metadata: + name: installerclusterdefault +spec: + snatIp: + - {{ aci_cni['cluster_snat_policy_ip'] }} From d26c039f610db7510c044135e010f32a771485ac Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Tue, 4 Aug 2020 12:21:47 -0700 Subject: [PATCH 17/48] Set MTU for node network Use the same MTU field from opflex network --- upi/openstack/02_network.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index c640e6b4866..b291c1e9dc8 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -167,3 +167,8 @@ - name: 'Attach the Ingress floating IP to Ingress port' command: cmd: "openstack floating ip set --port {{ os_port_ingress }} {{ os_ingress_fip }}" + + - name: 'Set MTU for the node network' + command: + cmd: "openstack networrk set {{ os_network }} --mtu {{ aci_cni['network_interfaces']['opflex']['mtu'] }}" + when: os_networking_type == "CiscoACI" From b163209a751acab01ea6ed008f52bc25e3875f52 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Wed, 5 Aug 2020 11:35:00 -0700 Subject: [PATCH 18/48] Fix minor typo in network Ansible task --- upi/openstack/02_network.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index b291c1e9dc8..8c85cf0eca4 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -170,5 +170,5 @@ - name: 'Set MTU for the node network' command: - cmd: "openstack networrk set {{ os_network }} --mtu {{ aci_cni['network_interfaces']['opflex']['mtu'] }}" + cmd: "openstack network set {{ os_network }} --mtu {{ aci_cni['network_interfaces']['opflex']['mtu'] }}" when: os_networking_type == "CiscoACI" From 8ddb074fbba0a3ef971780c3722d1bc0d9906c31 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Wed, 5 Aug 2020 19:02:56 -0700 Subject: [PATCH 19/48] Remove prefix-length arg from subnet creation task --- upi/openstack/02_network.yaml | 2 +- upi/openstack/update_ign.py | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index 8c85cf0eca4..c722100240e 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -40,7 +40,7 @@ - name: 'Create a subnet' command: - cmd: "openstack subnet create --network {{ os_network }} --subnet-pool node_network_subnet_pool --subnet-range {{ os_subnet_range }} --allocation-pool start={{ os_subnet_range | next_nth_usable(10) }},end={{ os_subnet_range | ipaddr('last_usable') }} --dhcp {{ os_subnet }} --prefix-length {{ aci_cni['network_interfaces']['node']['subnet_prefix_length'] }}" + cmd: "openstack subnet create --network {{ os_network }} --subnet-pool node_network_subnet_pool --subnet-range {{ os_subnet_range }} --allocation-pool start={{ os_subnet_range | next_nth_usable(10) }},end={{ os_subnet_range | ipaddr('last_usable') }} --dhcp {{ os_subnet }}" when: os_networking_type == "CiscoACI" - name: 'Set the cluster subnet tag' diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index 226571c1a8b..81dd6e0d51b 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -62,23 +62,12 @@ else: neutron_network_mtu = str(inventory['network_interfaces']['opflex']['mtu']) -# Returns the mask value for a subnet, for example 10.0.0.0/24 returns 24 -def get_prefix_from_subnet(subnet): - mask = None - try: - split_subnet = subnet.split("/") - mask = split_subnet[1] - except: - print("os_subnet_range not in valid format i.e a.b.c.d/e") - return mask - # Set infra_vlan field in inventory.yaml using accprovision tar value try: with open(original_inventory, 'r') as stream: cur_yaml = yaml.safe_load(stream) cur_yaml['all']['hosts']['localhost']['aci_cni']['infra_vlan'] = aci_infra_vlan cur_yaml['all']['hosts']['localhost']['aci_cni']['service_vlan'] = service_vlan - cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['subnet_prefix_length'] = get_prefix_from_subnet(os_subnet_range) cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['mtu'] = neutron_network_mtu if cur_yaml: From dce1f048e9d01b041bde2171d359b3f9c8d8855c Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Sat, 8 Aug 2020 11:20:10 -0700 Subject: [PATCH 20/48] Create node network with epg contract relationship --- upi/openstack/02_network.yaml | 6 ++++++ upi/openstack/inventory.yaml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index c722100240e..5c407c9fe0e 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -14,6 +14,12 @@ - name: 'Create the cluster network' os_network: name: "{{ os_network }}" + when: os_networking_type != "CiscoACI" + + - name: 'Create the cluster network with aci-containers-nodes EPG contract relationship for node network' + command: + cmd: "neutron net-create {{ os_network }} --apic:epg_contract_masters list=true type=dict app_profile_name={{ aci_cni['app_profile'] }},name={{ aci_cni['node_epg'] }}" + when: os_networking_type == "CiscoACI" - name: 'Set the cluster network tag' command: diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index df33d37abeb..ef93d99cbe4 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -35,6 +35,8 @@ all: os_compute_nodes_number: 3 aci_cni: kubeconfig: /path/to/kubeconfig + app_profile: app-profile + node_epg: net_123 acc_provision_tar: /home/noiro/openupi/aci_deployment.yaml.tar.gz network_interfaces: node: From 1b7d597967b0a519673d73e5ae341c46dc9df8f8 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Mon, 10 Aug 2020 14:37:59 -0700 Subject: [PATCH 21/48] Set primaryClusterNetwork tag on node network To distinguish the node network from the second kube network --- upi/openstack/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upi/openstack/common.yaml b/upi/openstack/common.yaml index 590b032541c..e3a92d8dd27 100644 --- a/upi/openstack/common.yaml +++ b/upi/openstack/common.yaml @@ -8,7 +8,7 @@ - name: 'Compute resource names' set_fact: cluster_id_tag: "openshiftClusterID={{ infraID }}" - os_network: "{{ infraID }}-network" + os_network: "{{ infraID }}-primaryClusterNetwork" os_subnet: "{{ infraID }}-nodes" os_router: "{{ infraID }}-external-router" # Port names From 9720b7fca59cb15b08c31dcb4f0e72342ac82fa1 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Mon, 10 Aug 2020 14:35:05 -0700 Subject: [PATCH 22/48] Load app_profile name from acc-provision tar file --- upi/openstack/update_ign.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index 81dd6e0d51b..be664a80a83 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -48,6 +48,7 @@ json_data = json.loads(data) aci_infra_vlan = json_data['aci-infra-vlan'] service_vlan = json_data['service-vlan'] + app_profile = json_data['app-profile'] except: print("Couldn't extract host-agent-config from aci-containers ConfigMap") @@ -66,6 +67,7 @@ try: with open(original_inventory, 'r') as stream: cur_yaml = yaml.safe_load(stream) + cur_yaml['all']['hosts']['localhost']['aci_cni']['app_profile'] = app_profile cur_yaml['all']['hosts']['localhost']['aci_cni']['infra_vlan'] = aci_infra_vlan cur_yaml['all']['hosts']['localhost']['aci_cni']['service_vlan'] = service_vlan cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['mtu'] = neutron_network_mtu From 848d5100666c69681868e6c0246ef94e96452bf4 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Tue, 11 Aug 2020 16:48:00 -0700 Subject: [PATCH 23/48] Add neutron network extension for node BD Also import the node bd and vrf dn from acc-provision tar gz --- upi/openstack/02_network.yaml | 2 +- upi/openstack/inventory.yaml | 73 ++++++++++++++--------------------- upi/openstack/update_ign.py | 19 +++++++-- 3 files changed, 46 insertions(+), 48 deletions(-) diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index 5c407c9fe0e..d1bb64b3213 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -18,7 +18,7 @@ - name: 'Create the cluster network with aci-containers-nodes EPG contract relationship for node network' command: - cmd: "neutron net-create {{ os_network }} --apic:epg_contract_masters list=true type=dict app_profile_name={{ aci_cni['app_profile'] }},name={{ aci_cni['node_epg'] }}" + cmd: "neutron net-create {{ os_network }} --apic:epg_contract_masters list=true type=dict app_profile_name={{ aci_cni['app_profile'] }},name={{ aci_cni['node_epg'] }} --apic:distinguished_names type=dict BridgeDomain={{ aci_cni['network_interfaces']['node']['bd']" when: os_networking_type == "CiscoACI" - name: 'Set the cluster network tag' diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index ef93d99cbe4..6d6a3d823d2 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -1,51 +1,38 @@ all: hosts: localhost: - ansible_connection: local - ansible_python_interpreter: "{{ansible_playbook_python}}" - - # User-provided values - os_subnet_range: '15.11.0.0/24' - os_flavor_master: 'aci_rhel_huge' - os_flavor_worker: 'aci_rhel_huge' - os_image_rhcos: 'rhcos-4.4' - os_external_network: 'l3out-2' - # OpenShift API floating IP address - os_api_fip: '60.60.60.6' - dns_ip: '172.28.184.18' - # OpenShift Ingress floating IP address - os_ingress_fip: '60.60.60.8' - # Service subnet cidr - svc_subnet_range: '172.30.0.0/16' - os_svc_network_range: '172.30.0.0/15' - # Subnet pool prefixes - cluster_network_cidrs: '15.128.0.0/14' - # Subnet pool prefix length - host_prefix: '23' - # Name of the SDN. - # Possible values are OpenshiftSDN or Kuryr. - os_networking_type: 'CiscoACI' - - # Number of provisioned Control Plane nodes - # 3 is the minimum number for a fully-functional cluster. - os_cp_nodes_number: 3 - - # Number of provisioned Compute nodes. - # 3 is the minimum number for a fully-functional cluster. - os_compute_nodes_number: 3 aci_cni: - kubeconfig: /path/to/kubeconfig - app_profile: app-profile - node_epg: net_123 - acc_provision_tar: /home/noiro/openupi/aci_deployment.yaml.tar.gz + # app_profile, infra_vlan, service_vlan, node network bd and vrf fields are derived from acc_provision_tar file + # opflex network mtu is set to 1500 if not provided + acc_provision_tar: /home/noiro/openupi/aci_deployment.yaml1.tar.gz + app_profile: aci-containers-openupi # OPTIONAL + cluster_snat_policy_ip: + infra_vlan: 4093 # OPTIONAL + kubeconfig: /home/noiro/openupi/openshift-env/upi/auth/kubeconfig network_interfaces: node: + bd: uni/tn-prj_/BD-aci-containers--node-bd # OPTIONAL name: ens3 - vrf: uni/tn-common/ctx-sauto_l3out-1_vrf - # no default MTU setting for node network - opflex: + vrf: uni/tn-/ctx- # OPTIONAL + opflex: + mtu: '1500' # OPTIONAL name: ens4 - subnet: '16.11.0.0/24' - mtu: 1500 - cluster_snat_policy_ip: - # generated + subnet: 16.11.0.0/24 + service_vlan: 1033 # OPTIONAL + ansible_connection: local + ansible_python_interpreter: '{{ansible_playbook_python}}' + cluster_network_cidrs: 15.128.0.0/16 + dns_ip: 172.28.184.18 + host_prefix: '23' + os_api_fip: 60.60.60.6 + os_compute_nodes_number: 2 + os_cp_nodes_number: 3 + os_external_network: l3out-2 + os_flavor_master: aci_rhel_huge + os_flavor_worker: aci_rhel_huge + os_image_rhcos: rhcos-4.4 + os_ingress_fip: 60.60.60.8 + os_networking_type: CiscoACI + os_subnet_range: 15.11.0.0/27 + os_svc_network_range: 172.30.0.0/15 + svc_subnet_range: 172.30.0.0/16 diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index be664a80a83..de2fd5aba47 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -42,13 +42,22 @@ data = yaml.safe_load(stream)['data']['host-agent-config'] except yaml.YAMLError as exc: print(exc) + with open(filepath, 'r') as stream: + try: + data_controller = yaml.safe_load(stream)['data']['controller-config'] + except yaml.YAMLError as exc: + print(exc) # Extract host-agent-config and obtain vlan values try: - json_data = json.loads(data) - aci_infra_vlan = json_data['aci-infra-vlan'] - service_vlan = json_data['service-vlan'] - app_profile = json_data['app-profile'] + host_agent_data = json.loads(data) + aci_infra_vlan = host_agent_data['aci-infra-vlan'] + service_vlan = host_agent_data['service-vlan'] + app_profile = host_agent_data['app-profile'] + + controller_data = json.loads(data_controller) + aci_vrf_dn = controller_data['aci-vrf-dn'] + aci_nodebd_dn = controller_data['aci-nodebd-dn'] except: print("Couldn't extract host-agent-config from aci-containers ConfigMap") @@ -71,6 +80,8 @@ cur_yaml['all']['hosts']['localhost']['aci_cni']['infra_vlan'] = aci_infra_vlan cur_yaml['all']['hosts']['localhost']['aci_cni']['service_vlan'] = service_vlan cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['mtu'] = neutron_network_mtu + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['vrf'] = aci_vrf_dn + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['bd'] = aci_nodebd_dn if cur_yaml: with open(processed_inventory,'w') as yamlfile: From dfe1018acb06714dd1a00d17276c14f3c7ed6bad Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Wed, 12 Aug 2020 18:33:29 -0700 Subject: [PATCH 24/48] Adding second interface configuration in worker machineset --- upi/openstack/update_ign.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index de2fd5aba47..8fae5302ed4 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -118,6 +118,28 @@ def update(hostname,ignition): }, 'filesystem': 'root', }) + for element in files: + if element["path"] == "/opt/openshift/openshift/99_openshift-cluster-api_worker-machineset-0.yaml": + ys_data = yaml.safe_load(base64.standard_b64decode(element["contents"]["source"].replace + ("data:text/plain;charset=utf-8;base64,", ''))) + networks = ys_data['spec']['template']['spec']['providerSpec']['value'].get('networks', []) + networks.append( + { + 'filter': {}, + 'subnets': [ + { + 'filter': { + 'name': str(infra_id.decode() + '-aci-containers-nodes'), + 'tags': str('openshiftClusterID=' + infra_id.decode()) + } + } + ] + } + ) + + ys_data['spec']['template']['spec']['providerSpec']['value']['networks'] = networks + element["contents"]["source"] = "data:text/plain;charset=utf-8;base64," + \ + base64.standard_b64encode(yaml.safe_dump(ys_data)).decode().strip() hostname_b64 = base64.standard_b64encode(hostname).decode().strip() files.append( From b86b5fb2f7d45acdb21fa59e7175a1b37d5276b2 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Thu, 13 Aug 2020 09:50:28 -0700 Subject: [PATCH 25/48] Missing brackets in network playbook and sample inventory file --- upi/openstack/02_network.yaml | 2 +- upi/openstack/inventory.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index d1bb64b3213..2d5ca92f269 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -18,7 +18,7 @@ - name: 'Create the cluster network with aci-containers-nodes EPG contract relationship for node network' command: - cmd: "neutron net-create {{ os_network }} --apic:epg_contract_masters list=true type=dict app_profile_name={{ aci_cni['app_profile'] }},name={{ aci_cni['node_epg'] }} --apic:distinguished_names type=dict BridgeDomain={{ aci_cni['network_interfaces']['node']['bd']" + cmd: "neutron net-create {{ os_network }} --apic:epg_contract_masters list=true type=dict app_profile_name={{ aci_cni['app_profile'] }},name={{ aci_cni['node_epg'] }} --apic:distinguished_names type=dict BridgeDomain={{ aci_cni['network_interfaces']['node']['bd'] }}" when: os_networking_type == "CiscoACI" - name: 'Set the cluster network tag' diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index 6d6a3d823d2..acf2efbe2c6 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -6,6 +6,7 @@ all: # opflex network mtu is set to 1500 if not provided acc_provision_tar: /home/noiro/openupi/aci_deployment.yaml1.tar.gz app_profile: aci-containers-openupi # OPTIONAL + node_epg: aci-containers-nodes cluster_snat_policy_ip: infra_vlan: 4093 # OPTIONAL kubeconfig: /home/noiro/openupi/openshift-env/upi/auth/kubeconfig From 3ea9cff41e4b431d422097cc13221d0d4c9e8fa0 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Thu, 13 Aug 2020 14:33:34 -0700 Subject: [PATCH 26/48] Add network scripts for control and compute nodes to bootstrap.ign --- .../templates/99_master-networkscripts.yaml | 42 +++++ .../templates/99_worker-networkscripts.yaml | 42 +++++ upi/openstack/update_ign.py | 165 +++++++++++------- 3 files changed, 189 insertions(+), 60 deletions(-) create mode 100644 upi/openstack/templates/99_master-networkscripts.yaml create mode 100644 upi/openstack/templates/99_worker-networkscripts.yaml diff --git a/upi/openstack/templates/99_master-networkscripts.yaml b/upi/openstack/templates/99_master-networkscripts.yaml new file mode 100644 index 00000000000..09f04157904 --- /dev/null +++ b/upi/openstack/templates/99_master-networkscripts.yaml @@ -0,0 +1,42 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 02-master-network +spec: + config: + ignition: + config: {} + security: + tls: {} + timeouts: {} + version: 2.2.0 + networkd: {} + passwd: {} + storage: + files: + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_ens3.base64 }} + verification: {} + path: {{ ifcfg_ens3.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_ens4.base64 }} + verification: {} + path: {{ ifcfg_ens4.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_opflex_conn.base64 }} + verification: {} + path: {{ ifcfg_opflex_conn.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ route_opflex_conn.base64 }} + verification: {} + path: {{ route_opflex_conn.path }} + mode: 420 + filesystem: root diff --git a/upi/openstack/templates/99_worker-networkscripts.yaml b/upi/openstack/templates/99_worker-networkscripts.yaml new file mode 100644 index 00000000000..aeaabed7651 --- /dev/null +++ b/upi/openstack/templates/99_worker-networkscripts.yaml @@ -0,0 +1,42 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: worker + name: 02-worker-network +spec: + config: + ignition: + config: {} + security: + tls: {} + timeouts: {} + version: 2.2.0 + networkd: {} + passwd: {} + storage: + files: + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_ens3.base64 }} + verification: {} + path: {{ ifcfg_ens3.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_ens4.base64 }} + verification: {} + path: {{ ifcfg_ens4.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ ifcfg_opflex_conn.base64 }} + verification: {} + path: {{ ifcfg_opflex_conn.path }} + mode: 420 + filesystem: root + - contents: + source: data:text/plain;charset=utf-8;base64,{{ route_opflex_conn.base64 }} + verification: {} + path: {{ route_opflex_conn.path }} + mode: 420 + filesystem: root diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index 8fae5302ed4..2a44b881f45 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -4,6 +4,7 @@ import shutil import tarfile import yaml +from jinja2 import Environment, FileSystemLoader #The script does the following things: @@ -100,6 +101,69 @@ infra_id = os.environ.get('INFRA_ID', 'openshift').encode() def update(hostname,ignition): + + config_data = {} + + ifcfg_ens3 = ("""TYPE=Ethernet + DEVICE=""" + node_interface + """ + ONBOOT=yes + BOOTPROTO=dhcp + DEFROUTE=yes + PROXY_METHOD=none + BROWSER_ONLY=no + MTU=""" + neutron_network_mtu + """ + IPV4_FAILURE_FATAL=no + IPV6INIT=no""").encode() + + ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip() + + config_data['ifcfg_ens3'] = {'base64': ifcfg_ens3_b64, 'path': '/etc/sysconfig/network-scripts/ifcfg-ens3'} + + ifcfg_ens4 = ("""TYPE=Ethernet + DEVICE=""" + opflex_interface + """ + ONBOOT=yes + BOOTPROTO=dhcp + DEFROUTE=no + PROXY_METHOD=none + BROWSER_ONLY=no + MTU=""" + neutron_network_mtu + """ + IPV4_FAILURE_FATAL=no + IPV6INIT=no""").encode() + + ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() + + config_data['ifcfg_ens4'] = {'base64': ifcfg_ens4_b64, 'path': '/etc/sysconfig/network-scripts/ifcfg-ens4'} + + opflex_conn = ("""VLAN=yes + TYPE=Vlan + PHYSDEV=""" + opflex_interface + """ + VLAN_ID=""" + infra_vlan + """ + REORDER_HDR=yes + GVRP=no + MVRP=no + PROXY_METHOD=none + BROWSER_ONLY=no + BOOTPROTO=dhcp + DEFROUTE=no + IPV4_FAILURE_FATAL=no + IPV6INIT=no + NAME=opflex-conn + DEVICE=""" + opflex_interface + """.""" + infra_vlan + """ + ONBOOT=yes + MTU=""" + neutron_network_mtu).encode() + + ifcfg_opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip() + + config_data['ifcfg_opflex_conn'] = {'base64': ifcfg_opflex_conn_b64, 'path': '/etc/sysconfig/network-scripts/ifcfg-opflex-conn'} + + route_opflex_conn = """ADDRESS0=224.0.0.0 + NETMASK0=240.0.0.0 + METRIC0=1000""".encode() + + route_opflex_conn_b64 = base64.standard_b64encode(route_opflex_conn).decode().strip() + + config_data['route_opflex_conn'] = {'base64': route_opflex_conn_b64, 'path': '/etc/sysconfig/network-scripts/route-opflex-conn'} + files = ignition['storage'].get('files', []) if 'bootstrap' in hostname.decode(): ca_cert_path = os.environ.get('OS_CACERT', '') @@ -118,6 +182,39 @@ def update(hostname,ignition): }, 'filesystem': 'root', }) + + # Add master and worker network scripts to bootstrap ignition + env = Environment(loader = FileSystemLoader('./templates'), trim_blocks=True, lstrip_blocks=True) + template_worker = env.get_template('99_worker-networkscripts.yaml') + rendered_worker = template_worker.render(config_data) + worker_b64 = base64.standard_b64encode(rendered_worker).decode().strip() + + template_master = env.get_template('99_master-networkscripts.yaml') + rendered_master = template_master.render(config_data) + master_b64 = base64.standard_b64encode(rendered_master).decode().strip() + + files.append( + { + 'path': '/opt/openshift/openshift/99_master-networkscripts.yaml', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + master_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': '/opt/openshift/openshift/99_worker-networkscripts.yaml', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + worker_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + for element in files: if element["path"] == "/opt/openshift/openshift/99_openshift-cluster-api_worker-machineset-0.yaml": ys_data = yaml.safe_load(base64.standard_b64decode(element["contents"]["source"].replace @@ -140,7 +237,6 @@ def update(hostname,ignition): ys_data['spec']['template']['spec']['providerSpec']['value']['networks'] = networks element["contents"]["source"] = "data:text/plain;charset=utf-8;base64," + \ base64.standard_b64encode(yaml.safe_dump(ys_data)).decode().strip() - hostname_b64 = base64.standard_b64encode(hostname).decode().strip() files.append( { @@ -152,97 +248,46 @@ def update(hostname,ignition): }, 'filesystem': 'root', }) - ifcfg_ens3 = ("""TYPE=Ethernet - DEVICE=""" + node_interface + """ - ONBOOT=yes - BOOTPROTO=dhcp - DEFROUTE=yes - PROXY_METHOD=none - BROWSER_ONLY=no - MTU=""" + neutron_network_mtu + """ - IPV4_FAILURE_FATAL=no - IPV6INIT=no""").encode() - - ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip() files.append( { - 'path': '/etc/sysconfig/network-scripts/ifcfg-ens3', + 'path': config_data['ifcfg_ens3']['path'], 'mode': 420, 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens3_b64, + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_ens3']['base64'], 'verification': {} }, 'filesystem': 'root', }) - ifcfg_ens4 = ("""TYPE=Ethernet - DEVICE=""" + opflex_interface + """ - ONBOOT=yes - BOOTPROTO=dhcp - DEFROUTE=no - PROXY_METHOD=none - BROWSER_ONLY=no - MTU=""" + neutron_network_mtu + """ - IPV4_FAILURE_FATAL=no - IPV6INIT=no""").encode() - - ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() - files.append( { - 'path': '/etc/sysconfig/network-scripts/ifcfg-ens4', + 'path': config_data['ifcfg_ens4']['path'], 'mode': 420, 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + ifcfg_ens4_b64, + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_ens4']['base64'], 'verification': {} }, 'filesystem': 'root', }) - opflex_conn = ("""VLAN=yes - TYPE=Vlan - PHYSDEV=""" + opflex_interface + """ - VLAN_ID=""" + infra_vlan + """ - REORDER_HDR=yes - GVRP=no - MVRP=no - PROXY_METHOD=none - BROWSER_ONLY=no - BOOTPROTO=dhcp - DEFROUTE=no - IPV4_FAILURE_FATAL=no - IPV6INIT=no - NAME=opflex-conn - DEVICE=""" + opflex_interface + """.""" + infra_vlan + """ - ONBOOT=yes - MTU=""" + neutron_network_mtu).encode() - - opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip() - files.append( { - 'path': '/etc/sysconfig/network-scripts/ifcfg-opflex-conn', + 'path': config_data['ifcfg_opflex_conn']['path'], 'mode': 420, 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + opflex_conn_b64, + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_opflex_conn']['base64'], 'verification': {} }, 'filesystem': 'root', }) - route_opflex_conn = """ADDRESS0=224.0.0.0 - NETMASK0=240.0.0.0 - METRIC0=1000""".encode() - - route_opflex_conn_b64 = base64.standard_b64encode(route_opflex_conn).decode().strip() - files.append( { - 'path': '/etc/sysconfig/network-scripts/route-opflex-conn', + 'path': config_data['route_opflex_conn']['path'], 'mode': 420, 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + route_opflex_conn_b64, + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['route_opflex_conn']['base64'], 'verification': {} }, 'filesystem': 'root', From 6897f746c7585065eec50692878d15fdf40ca3b9 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Fri, 14 Aug 2020 00:07:36 -0700 Subject: [PATCH 27/48] Set default node_epg --- upi/openstack/update_ign.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index 2a44b881f45..4c4da0cadab 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -73,12 +73,18 @@ else: neutron_network_mtu = str(inventory['network_interfaces']['opflex']['mtu']) +if 'node_epg' not in inventory: + node_epg = "aci-containers-nodes" +else: + node_epg = inventory['node_epg'] + # Set infra_vlan field in inventory.yaml using accprovision tar value try: with open(original_inventory, 'r') as stream: cur_yaml = yaml.safe_load(stream) cur_yaml['all']['hosts']['localhost']['aci_cni']['app_profile'] = app_profile cur_yaml['all']['hosts']['localhost']['aci_cni']['infra_vlan'] = aci_infra_vlan + cur_yaml['all']['hosts']['localhost']['aci_cni']['node_epg'] = node_epg cur_yaml['all']['hosts']['localhost']['aci_cni']['service_vlan'] = service_vlan cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['mtu'] = neutron_network_mtu cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['vrf'] = aci_vrf_dn From 8b2a348fec1549fec12c1b035398d162cb5dbfd4 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Tue, 18 Aug 2020 13:57:34 -0700 Subject: [PATCH 28/48] Set defaults for network interface fields Set defaults for the following fields in inventory.yaml file: node network interface to ens3 opflex network interface to ens4 opflex network subnet to 192.168.208.0/20 --- upi/openstack/inventory.yaml | 11 +++---- upi/openstack/update_ign.py | 57 ++++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index acf2efbe2c6..26c1ab57ae1 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -6,19 +6,20 @@ all: # opflex network mtu is set to 1500 if not provided acc_provision_tar: /home/noiro/openupi/aci_deployment.yaml1.tar.gz app_profile: aci-containers-openupi # OPTIONAL - node_epg: aci-containers-nodes + node_epg: aci-containers-nodes # OPTIONAL cluster_snat_policy_ip: infra_vlan: 4093 # OPTIONAL kubeconfig: /home/noiro/openupi/openshift-env/upi/auth/kubeconfig network_interfaces: node: bd: uni/tn-prj_/BD-aci-containers--node-bd # OPTIONAL - name: ens3 + name: ens3 # OPTIONAL vrf: uni/tn-/ctx- # OPTIONAL + mtu: 1500 # OPTIONAL opflex: - mtu: '1500' # OPTIONAL - name: ens4 - subnet: 16.11.0.0/24 + mtu: 1500 # OPTIONAL + name: ens4 # OPTIONAL + subnet: 192.168.0.0/16 # OPTIONAL service_vlan: 1033 # OPTIONAL ansible_connection: local ansible_python_interpreter: '{{ansible_playbook_python}}' diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index 4c4da0cadab..fa41200a7b2 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -68,27 +68,45 @@ except OSError as e: print ("Error: %s - %s." % (e.filename, e.strerror)) -if 'mtu' not in inventory['network_interfaces']['opflex']: - neutron_network_mtu = "1500" -else: - neutron_network_mtu = str(inventory['network_interfaces']['opflex']['mtu']) - -if 'node_epg' not in inventory: - node_epg = "aci-containers-nodes" -else: - node_epg = inventory['node_epg'] - # Set infra_vlan field in inventory.yaml using accprovision tar value try: with open(original_inventory, 'r') as stream: cur_yaml = yaml.safe_load(stream) - cur_yaml['all']['hosts']['localhost']['aci_cni']['app_profile'] = app_profile - cur_yaml['all']['hosts']['localhost']['aci_cni']['infra_vlan'] = aci_infra_vlan - cur_yaml['all']['hosts']['localhost']['aci_cni']['node_epg'] = node_epg - cur_yaml['all']['hosts']['localhost']['aci_cni']['service_vlan'] = service_vlan - cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['mtu'] = neutron_network_mtu - cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['vrf'] = aci_vrf_dn - cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['bd'] = aci_nodebd_dn + + cur_yaml['all']['hosts']['localhost']['aci_cni']['app_profile'] = app_profile + cur_yaml['all']['hosts']['localhost']['aci_cni']['infra_vlan'] = aci_infra_vlan + + if 'node_epg' not in inventory: + cur_yaml['all']['hosts']['localhost']['aci_cni']['node_epg'] = "aci-containers-nodes" + + cur_yaml['all']['hosts']['localhost']['aci_cni']['service_vlan'] = service_vlan + + if 'network_interfaces' not in inventory: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces'] = dict() + + if 'opflex' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex'] = dict() + + if 'node' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node'] = dict() + + if 'mtu' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['mtu'] = 1500 + + if 'mtu' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['mtu'] = 1500 + + if 'name' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['name'] = 'ens4' + + if 'name' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['name'] = 'ens3' + + if 'subnet' not in cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']: + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['subnet'] = '192.168.208.0/20' + + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['vrf'] = aci_vrf_dn + cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['bd'] = aci_nodebd_dn if cur_yaml: with open(processed_inventory,'w') as yamlfile: @@ -96,8 +114,8 @@ except: print("Unable to edit inventory.yaml") try: - node_interface = inventory['network_interfaces']['node']['name'] - opflex_interface = inventory['network_interfaces']['opflex']['name'] + node_interface = cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['name'] + opflex_interface = cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['opflex']['name'] master_count = localhost['os_cp_nodes_number'] worker_count = localhost['os_compute_nodes_number'] except: @@ -105,6 +123,7 @@ infra_vlan = str(aci_infra_vlan) infra_id = os.environ.get('INFRA_ID', 'openshift').encode() +neutron_network_mtu = str(cur_yaml['all']['hosts']['localhost']['aci_cni']['network_interfaces']['node']['mtu']) def update(hostname,ignition): From f924a7ea208bd3aaac32e5985b29aacec519f778 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Wed, 19 Aug 2020 09:56:01 -0700 Subject: [PATCH 29/48] Fix network task to set mtu mtu field was introduced for node network as well in https://github.com/noironetworks/openshift_installer/pull/85, so pick that value while setting the mtu now --- upi/openstack/02_network.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index 2d5ca92f269..d4a0069a465 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -176,5 +176,5 @@ - name: 'Set MTU for the node network' command: - cmd: "openstack network set {{ os_network }} --mtu {{ aci_cni['network_interfaces']['opflex']['mtu'] }}" + cmd: "openstack network set {{ os_network }} --mtu {{ aci_cni['network_interfaces']['node']['mtu'] }}" when: os_networking_type == "CiscoACI" From b7a264270014d1c72266fcee386918a1bb0f4015 Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Fri, 21 Aug 2020 08:57:21 -0700 Subject: [PATCH 30/48] Renaming the acicontainers network in alphabetical order to come after {INFRA-ID}-primaryClusterNetwork --- upi/openstack/common_ciscoaci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upi/openstack/common_ciscoaci.yaml b/upi/openstack/common_ciscoaci.yaml index 8bb06d4f57c..b35562ac73d 100644 --- a/upi/openstack/common_ciscoaci.yaml +++ b/upi/openstack/common_ciscoaci.yaml @@ -1,6 +1,6 @@ - name: 'Compute CiscoACI resource names' set_fact: - os_aci_containers_network: "{{ infraID }}-acicontainers-network" + os_aci_containers_network: "{{ infraID }}-secondaryClusterNetwork-acicontainers" os_aci_containers_subnet: "{{ infraID }}-acicontainers-nodes" os_aci_containers_port_bootstrap: "{{ infraID }}-acicontainers-bootstrap-port" os_aci_containers_port_master: "{{ infraID }}-acicontainers-master-port" From 1caa5c9aa7289ea8007b28eb6f37a594127429c7 Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Fri, 21 Aug 2020 13:05:17 -0700 Subject: [PATCH 31/48] Adding MachineConfig to disable MCO Disk Validation Check --- ...9_worker-disable-mco-validation-check.yaml | 29 +++++++++++++++++++ upi/openstack/update_ign.py | 15 ++++++++++ 2 files changed, 44 insertions(+) create mode 100644 upi/openstack/files/99_worker-disable-mco-validation-check.yaml diff --git a/upi/openstack/files/99_worker-disable-mco-validation-check.yaml b/upi/openstack/files/99_worker-disable-mco-validation-check.yaml new file mode 100644 index 00000000000..b4d39a0d578 --- /dev/null +++ b/upi/openstack/files/99_worker-disable-mco-validation-check.yaml @@ -0,0 +1,29 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: worker + name: 02-worker-mco-check-disable +spec: + config: + ignition: + config: {} + security: + tls: {} + timeouts: {} + version: 2.2.0 + networkd: {} + passwd: {} + storage: {} + systemd: + units: + - contents: | + [Unit] + Description=Disabling Machine-Config Validation Check + [Service] + Type=simple + ExecStart=sh -c "while true; do sudo touch /run/machine-config-daemon-force; sleep 1; done" + [Install] + WantedBy=multi-user.target + enabled: true + name: machine-config-daemon-force.service diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index fa41200a7b2..1eaf8352ffa 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -218,6 +218,10 @@ def update(hostname,ignition): rendered_master = template_master.render(config_data) master_b64 = base64.standard_b64encode(rendered_master).decode().strip() + with open('./files/99_worker-disable-mco-validation-check.yaml', 'r') as f: + mc_config = f.read().encode() + mc_config_b64 = base64.standard_b64encode(mc_config).decode().strip() + files.append( { 'path': '/opt/openshift/openshift/99_master-networkscripts.yaml', @@ -240,6 +244,17 @@ def update(hostname,ignition): 'filesystem': 'root', }) + files.append( + { + 'path': '/opt/openshift/openshift/99_worker-disable-mco-validation-check.yaml', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + mc_config_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + for element in files: if element["path"] == "/opt/openshift/openshift/99_openshift-cluster-api_worker-machineset-0.yaml": ys_data = yaml.safe_load(base64.standard_b64decode(element["contents"]["source"].replace From a806a613d925ed8171a6e10432fc5384c03ca49c Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Tue, 25 Aug 2020 20:31:25 -0700 Subject: [PATCH 32/48] Adding functionality to create the bootstrap shim ignition file --- upi/openstack/update_ign.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index 1eaf8352ffa..ab349617f6f 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -344,6 +344,28 @@ def update(hostname,ignition): with open('bootstrap.ign', 'w') as f: json.dump(ignition, f) +os.system('''cat > $INFRA_ID-bootstrap-ignition.json << EOL +{ + "ignition": { + "config": { + "append": [ + { + "source": "$(swift stat -v | grep StorageURL | awk -F': ' '{print$2}')/bootstrap/bootstrap.ign", + "verification": {} + } + ] + }, + "security": {}, + "timeouts": {}, + "version": "2.2.0" + }, + "networkd": {}, + "passwd": {}, + "storage": {}, + "systemd": {} +} +EOL''') + for index in range(0,master_count): master_hostname = infra_id + b'-master-' + str(index).encode() + b'\n' with open('master.ign', 'r') as f: From 6ee19ea55a7ec293962e1ed92c57b0e64a1ab8e7 Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Thu, 27 Aug 2020 23:51:57 -0700 Subject: [PATCH 33/48] Adding the post-install steps for the Cluster and renaming files --- upi/openstack/cluster_snat_policy.yaml | 14 ++++++++++++++ .../files/cluster_endpoint_strategy.json | 16 ++++++++++++++++ upi/openstack/post-install.yaml | 13 +++---------- 3 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 upi/openstack/cluster_snat_policy.yaml create mode 100644 upi/openstack/files/cluster_endpoint_strategy.json diff --git a/upi/openstack/cluster_snat_policy.yaml b/upi/openstack/cluster_snat_policy.yaml new file mode 100644 index 00000000000..e3895756674 --- /dev/null +++ b/upi/openstack/cluster_snat_policy.yaml @@ -0,0 +1,14 @@ +- hosts: all + gather_facts: no + + tasks: + - name: 'Create snat policy file' + template: + src: cluster_snat_policy.conf.j2 + dest: cluster_snatpolicy.yaml + + - name: 'Create cluster SNAT policy' + k8s: + state: present + kubeconfig: "{{ aci_cni['kubeconfig'] }}" + src: cluster_snatpolicy.yaml diff --git a/upi/openstack/files/cluster_endpoint_strategy.json b/upi/openstack/files/cluster_endpoint_strategy.json new file mode 100644 index 00000000000..f012b5d10ba --- /dev/null +++ b/upi/openstack/files/cluster_endpoint_strategy.json @@ -0,0 +1,16 @@ +{ + "apiVersion": "operator.openshift.io/v1", + "kind": "IngressController", + "metadata": { + "namespace": "openshift-ingress-operator", + "name": "default" + }, + "spec": { + "endpointPublishingStrategy": { + "type": "LoadBalancerService", + "loadBalancer": { + "scope": "Internal" + } + } + } +} diff --git a/upi/openstack/post-install.yaml b/upi/openstack/post-install.yaml index e3895756674..f7ec45124c5 100644 --- a/upi/openstack/post-install.yaml +++ b/upi/openstack/post-install.yaml @@ -2,13 +2,6 @@ gather_facts: no tasks: - - name: 'Create snat policy file' - template: - src: cluster_snat_policy.conf.j2 - dest: cluster_snatpolicy.yaml - - - name: 'Create cluster SNAT policy' - k8s: - state: present - kubeconfig: "{{ aci_cni['kubeconfig'] }}" - src: cluster_snatpolicy.yaml + - name: 'Replace cluster endpoint stratergy to Loadbalancer' + command: + cmd: "oc replace --force --wait -f files/cluster_endpoint_strategy.json" From 8cc0621e0c4c5df12e3990188d18a05e0357030a Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Fri, 28 Aug 2020 14:55:12 -0700 Subject: [PATCH 34/48] Move dns_ip field under aci_cni and make related task conditional Add dns_ip under aci_cni in inventory file, however this is optional --- upi/openstack/021_network.yaml | 6 ++++-- upi/openstack/inventory.yaml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml index 9065278ffc4..26c28b431ce 100644 --- a/upi/openstack/021_network.yaml +++ b/upi/openstack/021_network.yaml @@ -36,5 +36,7 @@ - name: 'Set dns nameserver' command: - cmd: "openstack subnet set --dns-nameserver {{ dns_ip }} {{ os_subnet }}" - when: os_networking_type == "CiscoACI" + cmd: "openstack subnet set --dns-nameserver {{ aci_cni['dns_ip'] }} {{ os_subnet }}" + when: + - os_networking_type == "CiscoACI" + - aci_cni['dns_ip'] is defined diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index 26c1ab57ae1..fa316825ecf 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -8,6 +8,7 @@ all: app_profile: aci-containers-openupi # OPTIONAL node_epg: aci-containers-nodes # OPTIONAL cluster_snat_policy_ip: + dns_ip: 172.28.184.18 # OPTIONAL infra_vlan: 4093 # OPTIONAL kubeconfig: /home/noiro/openupi/openshift-env/upi/auth/kubeconfig network_interfaces: @@ -24,7 +25,6 @@ all: ansible_connection: local ansible_python_interpreter: '{{ansible_playbook_python}}' cluster_network_cidrs: 15.128.0.0/16 - dns_ip: 172.28.184.18 host_prefix: '23' os_api_fip: 60.60.60.6 os_compute_nodes_number: 2 From 45e7d39c979249d8f913d1fc06d3330c2e875460 Mon Sep 17 00:00:00 2001 From: Sumit Naiksatam Date: Fri, 28 Aug 2020 21:32:02 -0700 Subject: [PATCH 35/48] Clarifying sample config in inventory.yaml --- upi/openstack/inventory.yaml | 40 +++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index fa316825ecf..eadeff857b5 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -1,27 +1,33 @@ all: hosts: localhost: + ####################################################################### aci_cni: - # app_profile, infra_vlan, service_vlan, node network bd and vrf fields are derived from acc_provision_tar file - # opflex network mtu is set to 1500 if not provided - acc_provision_tar: /home/noiro/openupi/aci_deployment.yaml1.tar.gz - app_profile: aci-containers-openupi # OPTIONAL - node_epg: aci-containers-nodes # OPTIONAL - cluster_snat_policy_ip: - dns_ip: 172.28.184.18 # OPTIONAL - infra_vlan: 4093 # OPTIONAL - kubeconfig: /home/noiro/openupi/openshift-env/upi/auth/kubeconfig + # This section has ACI CNI specific configuration. + # In case of optional configuration, the default + # values are populated after running update_ign.py. + # Those default values are mentioned here. + acc_provision_tar: /path/aci_deployment.yaml1.tar.gz + kubeconfig: /path/kubeconfig network_interfaces: node: - bd: uni/tn-prj_/BD-aci-containers--node-bd # OPTIONAL - name: ens3 # OPTIONAL - vrf: uni/tn-/ctx- # OPTIONAL - mtu: 1500 # OPTIONAL + name: ens3 # Optional + mtu: 1500 # Optional + bd: uni/tn-prj_/BD-aci-containers--node-bd # Derived + vrf: uni/tn-/ctx- # Derived opflex: - mtu: 1500 # OPTIONAL - name: ens4 # OPTIONAL - subnet: 192.168.0.0/16 # OPTIONAL - service_vlan: 1033 # OPTIONAL + mtu: 1500 # Optional + name: ens4 # Optional + subnet: 192.168.208.0/20 # Optional + cluster_snat_policy_ip: # Optional, no default value + dns_ip: # Optional, no default value + # The following are derived from acc_provision_tar file + # and populated after running the update_ign.py + app_profile: aci-containers- # Derived + node_epg: aci-containers-nodes # Derived + infra_vlan: 4093 # Derived + service_vlan: 1033 # Derived + ####################################################################### ansible_connection: local ansible_python_interpreter: '{{ansible_playbook_python}}' cluster_network_cidrs: 15.128.0.0/16 From 78c7042752d8ea6c1372681ce73aeb56c1e64815 Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Sun, 30 Aug 2020 01:39:43 -0700 Subject: [PATCH 36/48] Adding relevant changes to file update_ign.py for running it via python3 version --- upi/openstack/update_ign.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index ab349617f6f..2887a94aa83 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -212,11 +212,11 @@ def update(hostname,ignition): env = Environment(loader = FileSystemLoader('./templates'), trim_blocks=True, lstrip_blocks=True) template_worker = env.get_template('99_worker-networkscripts.yaml') rendered_worker = template_worker.render(config_data) - worker_b64 = base64.standard_b64encode(rendered_worker).decode().strip() + worker_b64 = base64.standard_b64encode(rendered_worker.encode()).decode().strip() template_master = env.get_template('99_master-networkscripts.yaml') rendered_master = template_master.render(config_data) - master_b64 = base64.standard_b64encode(rendered_master).decode().strip() + master_b64 = base64.standard_b64encode(rendered_master.encode()).decode().strip() with open('./files/99_worker-disable-mco-validation-check.yaml', 'r') as f: mc_config = f.read().encode() @@ -276,7 +276,7 @@ def update(hostname,ignition): ys_data['spec']['template']['spec']['providerSpec']['value']['networks'] = networks element["contents"]["source"] = "data:text/plain;charset=utf-8;base64," + \ - base64.standard_b64encode(yaml.safe_dump(ys_data)).decode().strip() + base64.standard_b64encode(yaml.safe_dump(ys_data).encode()).decode().strip() hostname_b64 = base64.standard_b64encode(hostname).decode().strip() files.append( { @@ -344,7 +344,7 @@ def update(hostname,ignition): with open('bootstrap.ign', 'w') as f: json.dump(ignition, f) -os.system('''cat > $INFRA_ID-bootstrap-ignition.json << EOL +os.system('cat > ' + infra_id.decode() + '''-bootstrap-ignition.json << EOL { "ignition": { "config": { From de637b2cdd6cb0d3ee7c5c8b2fc3b6006be0c873 Mon Sep 17 00:00:00 2001 From: Apoorva Mittal Date: Thu, 10 Sep 2020 15:55:53 -0700 Subject: [PATCH 37/48] Delete .down-02_network.yaml.swp Included by mistake --- upi/openstack/.down-02_network.yaml.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 upi/openstack/.down-02_network.yaml.swp diff --git a/upi/openstack/.down-02_network.yaml.swp b/upi/openstack/.down-02_network.yaml.swp deleted file mode 100644 index a9b31ac43f92bd06ee88f19dff7f191c454e58db..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2ONbmr7{@Dt_)e6KpvNz}V7&D1&L$A;xI&1R;DflZxwbXkH8Yjz>6)#ona!{y zIeQlL6fZswISGMy&B=oxih76_1qtdw#iOU-zq)&NdS)1A)(4_g!;hY>uCB-L`?_Y9 zt@`?<3$N0d)zb{e6O7&cjiOKEW9*%-kHDh2E2ihfu3yy&mG-9wNYDlg)7S^==jq1=)V8t4R{0IfH&X`cmv*m zH{cC;1OIyiwtI+O$1D$*Grd@z4^5oQOMmhPya8{(8}J6a0dK$?@CLjAZ@?Sy2E2j) zpaGsRwsVBB&mO?w@&Etf-~aa>XY6}$7kmOf2DiaG;BD|G=z|_e!8vdm+&RYB2jG3M z2Qu(1`1L4bzkoa77N~*aUz*!K2C&3Z$ z<0FiH1-=9yf%m``I156s1P+6r9%k%o@GiIkENFsf!2O39`wn~yZh;TMW$*@g0Xz+! z0zW^9df+Cw4z7XM!AszGcyumrJD>jW2D|}pz#I6x4G@mqmdHXz+fQ9Vm1;Ie2u7zJQl^~*gy&_$hmf-{U) zdaaDV(j^_xaHYX=8d@}^lOIk^XTJW*ENCiQDqT^EE+`0G`2#w1FSn`Yk@5i;5$3whm$%$D|x#B@))hE{7sxBD+q zdtL$$d!2GSyfGurT8b_B;2XfcqSP?J;YG~D=vAB-bjEUhc zATtqt8H)frl!e|b^D-PYqeVBBG2TtQ%^5PV!DA$QuxJS8L|}hT+~OOpB@(h~d}_1e zQq2Ef56b+AR0sNJ!%?>J4+NkXzq4;A^%Gfc+9q^g`%R8jUq=YO#d@A87EntW_Zi#W z8Lni Date: Wed, 16 Sep 2020 10:24:53 +0200 Subject: [PATCH 38/48] make down network tasks more resilient --- upi/openstack/down-02_network.yaml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/upi/openstack/down-02_network.yaml b/upi/openstack/down-02_network.yaml index 7b769976ce8..e5376058052 100644 --- a/upi/openstack/down-02_network.yaml +++ b/upi/openstack/down-02_network.yaml @@ -69,12 +69,29 @@ - os_networking_type == "Kuryr" - pods_subnet_pool.stdout != "" + - name: 'List the cluster subnet pool' + command: + cmd: "openstack subnet pool list --name node_network_subnet_pool" + when: os_networking_type == "CiscoACI" + register: cisco_subnet_pool + - name: 'Remove the cluster subnet pool' command: cmd: "openstack subnet pool delete node_network_subnet_pool" + when: + - os_networking_type == "CiscoACI" + - cisco_subnet_pool.stdout != "" + + - name: 'List the cluster address-scope' + command: + cmd: "openstack subnet pool list --name node_network_subnet_pool" when: os_networking_type == "CiscoACI" + register: cisco_cluster_address_scope - name: 'Remove the cluster address-scope' command: cmd: "openstack address scope delete node_network_address_scope" - when: os_networking_type == "CiscoACI" + when: + - os_networking_type == "CiscoACI" + - cisco_cluster_address_scope.stdout != "" + From 5ef9b845eac92d8c2ba5ef3cd442c388f7b0a2f8 Mon Sep 17 00:00:00 2001 From: Janine Eichler Date: Wed, 16 Sep 2020 11:17:02 +0200 Subject: [PATCH 39/48] fix listing of cluster address scope in down network playbook --- upi/openstack/down-02_network.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upi/openstack/down-02_network.yaml b/upi/openstack/down-02_network.yaml index e5376058052..bc88c5343df 100644 --- a/upi/openstack/down-02_network.yaml +++ b/upi/openstack/down-02_network.yaml @@ -84,7 +84,7 @@ - name: 'List the cluster address-scope' command: - cmd: "openstack subnet pool list --name node_network_subnet_pool" + cmd: "openstack address scope list --name node_network_address_scope" when: os_networking_type == "CiscoACI" register: cisco_cluster_address_scope From 9ace8dc80b840b37ef208b60da4995fa03963462 Mon Sep 17 00:00:00 2001 From: Janine Eichler Date: Wed, 16 Sep 2020 14:12:35 +0200 Subject: [PATCH 40/48] directly use k8s to create the cluster snat policy from a template --- upi/openstack/cluster_snat_policy.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/upi/openstack/cluster_snat_policy.yaml b/upi/openstack/cluster_snat_policy.yaml index e3895756674..c19ad5073f2 100644 --- a/upi/openstack/cluster_snat_policy.yaml +++ b/upi/openstack/cluster_snat_policy.yaml @@ -1,14 +1,10 @@ - hosts: all gather_facts: no - tasks: - - name: 'Create snat policy file' - template: - src: cluster_snat_policy.conf.j2 - dest: cluster_snatpolicy.yaml - name: 'Create cluster SNAT policy' k8s: state: present kubeconfig: "{{ aci_cni['kubeconfig'] }}" - src: cluster_snatpolicy.yaml + defintion: "{{ lookup('template', 'cluster_snat_policy.conf.j2') }}" + From 101a46c93336d38e094620165f0432f4a63d47cc Mon Sep 17 00:00:00 2001 From: Janine Eichler Date: Wed, 16 Sep 2020 14:14:29 +0200 Subject: [PATCH 41/48] fix typo --- upi/openstack/cluster_snat_policy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upi/openstack/cluster_snat_policy.yaml b/upi/openstack/cluster_snat_policy.yaml index c19ad5073f2..733304d858e 100644 --- a/upi/openstack/cluster_snat_policy.yaml +++ b/upi/openstack/cluster_snat_policy.yaml @@ -6,5 +6,5 @@ k8s: state: present kubeconfig: "{{ aci_cni['kubeconfig'] }}" - defintion: "{{ lookup('template', 'cluster_snat_policy.conf.j2') }}" + definition: "{{ lookup('template', 'cluster_snat_policy.conf.j2') }}" From de07e9db7445a4ab058732efc1b81efa8d595079 Mon Sep 17 00:00:00 2001 From: Janine Eichler Date: Wed, 16 Sep 2020 22:02:26 +0200 Subject: [PATCH 42/48] add support for multiple dns ips --- upi/openstack/021_network.yaml | 5 +++-- upi/openstack/inventory.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml index 26c28b431ce..0718a9fcaf3 100644 --- a/upi/openstack/021_network.yaml +++ b/upi/openstack/021_network.yaml @@ -36,7 +36,8 @@ - name: 'Set dns nameserver' command: - cmd: "openstack subnet set --dns-nameserver {{ aci_cni['dns_ip'] }} {{ os_subnet }}" + cmd: "openstack subnet set --dns-nameserver {{ item }} {{ os_subnet }}" when: - os_networking_type == "CiscoACI" - - aci_cni['dns_ip'] is defined + - aci_cni.dns_ips is defined and aci_cni.dns_ips | length > 0 + with_items: "{{ aci_cni.dns_ips }}" diff --git a/upi/openstack/inventory.yaml b/upi/openstack/inventory.yaml index eadeff857b5..f7f3baea68a 100644 --- a/upi/openstack/inventory.yaml +++ b/upi/openstack/inventory.yaml @@ -20,7 +20,7 @@ all: name: ens4 # Optional subnet: 192.168.208.0/20 # Optional cluster_snat_policy_ip: # Optional, no default value - dns_ip: # Optional, no default value + dns_ips: [] # Optional, no default value # The following are derived from acc_provision_tar file # and populated after running the update_ign.py app_profile: aci-containers- # Derived From ec777b498ff0429581af3ccc667d737959daee25 Mon Sep 17 00:00:00 2001 From: Janine Eichler Date: Wed, 16 Sep 2020 23:40:11 +0200 Subject: [PATCH 43/48] use k8s for ingresscontroller --- .../files/cluster_endpoint_strategy.json | 16 ---------------- ...ngresscontroller_internal_loadbalancer.yaml | 10 ++++++++++ upi/openstack/post-install.yaml | 18 +++++++++++++++--- 3 files changed, 25 insertions(+), 19 deletions(-) delete mode 100644 upi/openstack/files/cluster_endpoint_strategy.json create mode 100644 upi/openstack/files/ingresscontroller_internal_loadbalancer.yaml diff --git a/upi/openstack/files/cluster_endpoint_strategy.json b/upi/openstack/files/cluster_endpoint_strategy.json deleted file mode 100644 index f012b5d10ba..00000000000 --- a/upi/openstack/files/cluster_endpoint_strategy.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "apiVersion": "operator.openshift.io/v1", - "kind": "IngressController", - "metadata": { - "namespace": "openshift-ingress-operator", - "name": "default" - }, - "spec": { - "endpointPublishingStrategy": { - "type": "LoadBalancerService", - "loadBalancer": { - "scope": "Internal" - } - } - } -} diff --git a/upi/openstack/files/ingresscontroller_internal_loadbalancer.yaml b/upi/openstack/files/ingresscontroller_internal_loadbalancer.yaml new file mode 100644 index 00000000000..154517e0dc4 --- /dev/null +++ b/upi/openstack/files/ingresscontroller_internal_loadbalancer.yaml @@ -0,0 +1,10 @@ +apiVersion: operator.openshift.io/v1 +kind: IngressController +metadata: + namespace: openshift-ingress-operator + name: default +spec: + endpointPublishingStrategy: + type: LoadBalancerService + loadBalancer: + scope: Internal diff --git a/upi/openstack/post-install.yaml b/upi/openstack/post-install.yaml index f7ec45124c5..25a35483d82 100644 --- a/upi/openstack/post-install.yaml +++ b/upi/openstack/post-install.yaml @@ -2,6 +2,18 @@ gather_facts: no tasks: - - name: 'Replace cluster endpoint stratergy to Loadbalancer' - command: - cmd: "oc replace --force --wait -f files/cluster_endpoint_strategy.json" + - name: 'Remove to be replaced default ingress controller' + k8s: + state: absent + kubeconfig: "{{ aci_cni['kubeconfig'] }}" + force: yes + wait: yes + definition: "{{ lookup('file', 'files/ingresscontroller_internal_loadbalancer.yaml') }}" + + - name: 'Replace cluster endpoint strategy to Loadbalancer' + k8s: + state: present + kubeconfig: "{{ aci_cni['kubeconfig'] }}" + force: yes + wait: no + definition: "{{ lookup('file', 'files/ingresscontroller_internal_loadbalancer.yaml') }}" From 36c196fa992f152d274f3bec97e5e193fdb7a78a Mon Sep 17 00:00:00 2001 From: Noiro Date: Wed, 16 Sep 2020 18:36:07 -0400 Subject: [PATCH 44/48] Correct the subnet name when setting dns ip for secondary network --- upi/openstack/021_network.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upi/openstack/021_network.yaml b/upi/openstack/021_network.yaml index 26c28b431ce..1fd89439aa9 100644 --- a/upi/openstack/021_network.yaml +++ b/upi/openstack/021_network.yaml @@ -36,7 +36,7 @@ - name: 'Set dns nameserver' command: - cmd: "openstack subnet set --dns-nameserver {{ aci_cni['dns_ip'] }} {{ os_subnet }}" + cmd: "openstack subnet set --dns-nameserver {{ aci_cni['dns_ip'] }} {{ os_aci_containers_subnet }}" when: - os_networking_type == "CiscoACI" - aci_cni['dns_ip'] is defined From a0444f69b6ef3629e8ad701d8440e362f28a6b72 Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Wed, 16 Sep 2020 15:47:52 -0700 Subject: [PATCH 45/48] Moving the functionality to update the 99_openshift-cluster-api_worker-machineset-0.yaml before create ignition phase --- upi/openstack/update_ign.py | 22 ---------------------- upi/openstack/update_machineset.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 22 deletions(-) create mode 100644 upi/openstack/update_machineset.py diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index 2887a94aa83..374ca818ac1 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -255,28 +255,6 @@ def update(hostname,ignition): 'filesystem': 'root', }) - for element in files: - if element["path"] == "/opt/openshift/openshift/99_openshift-cluster-api_worker-machineset-0.yaml": - ys_data = yaml.safe_load(base64.standard_b64decode(element["contents"]["source"].replace - ("data:text/plain;charset=utf-8;base64,", ''))) - networks = ys_data['spec']['template']['spec']['providerSpec']['value'].get('networks', []) - networks.append( - { - 'filter': {}, - 'subnets': [ - { - 'filter': { - 'name': str(infra_id.decode() + '-aci-containers-nodes'), - 'tags': str('openshiftClusterID=' + infra_id.decode()) - } - } - ] - } - ) - - ys_data['spec']['template']['spec']['providerSpec']['value']['networks'] = networks - element["contents"]["source"] = "data:text/plain;charset=utf-8;base64," + \ - base64.standard_b64encode(yaml.safe_dump(ys_data).encode()).decode().strip() hostname_b64 = base64.standard_b64encode(hostname).decode().strip() files.append( { diff --git a/upi/openstack/update_machineset.py b/upi/openstack/update_machineset.py new file mode 100644 index 00000000000..107d2fad1b2 --- /dev/null +++ b/upi/openstack/update_machineset.py @@ -0,0 +1,19 @@ +import yaml; +path = "openshift/99_openshift-cluster-api_worker-machineset-0.yaml" +data = yaml.safe_load(open(path)); +networks = data['spec']['template']['spec']['providerSpec']['value'].get('networks', []) +networks.append( + { + 'filter': {}, + 'subnets': [ + { + 'filter': { + 'name': str(data['metadata']['labels']['machine.openshift.io/cluster-api-cluster'] + '-acicontainers-nodes'), + 'tags': str('openshiftClusterID=' + data['metadata']['labels']['machine.openshift.io/cluster-api-cluster']) + } + } + ] + } +) +data['spec']['template']['spec']['providerSpec']['value']['networks'] = networks +open(path, "w").write(yaml.dump(data, default_flow_style=False)) From 753b4a68eb825f9181837deaf96b012381d0baf4 Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Thu, 17 Sep 2020 01:14:46 -0700 Subject: [PATCH 46/48] Adding primaryClusterNetwork tag to primary network --- upi/openstack/02_network.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/upi/openstack/02_network.yaml b/upi/openstack/02_network.yaml index d4a0069a465..a808ab98e73 100644 --- a/upi/openstack/02_network.yaml +++ b/upi/openstack/02_network.yaml @@ -25,6 +25,10 @@ command: cmd: "openstack network set --tag {{ cluster_id_tag }} {{ os_network }}" + - name: 'Set the primaryClusterNetwork tag' + command: + cmd: "openstack network set --tag {{ os_network }} {{ os_network }}" + - name: 'Create a subnet' os_subnet: name: "{{ os_subnet }}" From 7826cfdfd335d74e379430a4fe87d457cb2c70ad Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Mon, 11 Jan 2021 12:29:27 +0530 Subject: [PATCH 47/48] Removing the dependency of disabling MCO check 1. Updated the updatee_ign.py with contents in network scripts as expected by the MCO 2. Removed MCO disable check files/yamls from the repository --- ...9_worker-disable-mco-validation-check.yaml | 29 ----- upi/openstack/update_ign.py | 100 +++++++++--------- 2 files changed, 49 insertions(+), 80 deletions(-) delete mode 100644 upi/openstack/files/99_worker-disable-mco-validation-check.yaml diff --git a/upi/openstack/files/99_worker-disable-mco-validation-check.yaml b/upi/openstack/files/99_worker-disable-mco-validation-check.yaml deleted file mode 100644 index b4d39a0d578..00000000000 --- a/upi/openstack/files/99_worker-disable-mco-validation-check.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 02-worker-mco-check-disable -spec: - config: - ignition: - config: {} - security: - tls: {} - timeouts: {} - version: 2.2.0 - networkd: {} - passwd: {} - storage: {} - systemd: - units: - - contents: | - [Unit] - Description=Disabling Machine-Config Validation Check - [Service] - Type=simple - ExecStart=sh -c "while true; do sudo touch /run/machine-config-daemon-force; sleep 1; done" - [Install] - WantedBy=multi-user.target - enabled: true - name: machine-config-daemon-force.service diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index 374ca818ac1..fac37917305 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -130,60 +130,73 @@ def update(hostname,ignition): config_data = {} ifcfg_ens3 = ("""TYPE=Ethernet - DEVICE=""" + node_interface + """ - ONBOOT=yes - BOOTPROTO=dhcp - DEFROUTE=yes - PROXY_METHOD=none - BROWSER_ONLY=no - MTU=""" + neutron_network_mtu + """ - IPV4_FAILURE_FATAL=no - IPV6INIT=no""").encode() +DEVICE=""" + node_interface + """ +ONBOOT=yes +BOOTPROTO=dhcp +DEFROUTE=yes +PROXY_METHOD=none +BROWSER_ONLY=no +MTU=""" + neutron_network_mtu + """ +IPV4_FAILURE_FATAL=no +IPV6INIT=no +ETHTOOL_OPTS="-K ens3 tx-checksum-ip-generic off" +NAME="System ens3" +UUID=21d47e65-8523-1a06-af22-6f121086f085 +""").encode() ifcfg_ens3_b64 = base64.standard_b64encode(ifcfg_ens3).decode().strip() config_data['ifcfg_ens3'] = {'base64': ifcfg_ens3_b64, 'path': '/etc/sysconfig/network-scripts/ifcfg-ens3'} ifcfg_ens4 = ("""TYPE=Ethernet - DEVICE=""" + opflex_interface + """ - ONBOOT=yes - BOOTPROTO=dhcp - DEFROUTE=no - PROXY_METHOD=none - BROWSER_ONLY=no - MTU=""" + neutron_network_mtu + """ - IPV4_FAILURE_FATAL=no - IPV6INIT=no""").encode() +DEVICE=""" + opflex_interface + """ +ONBOOT=yes +BOOTPROTO=dhcp +DEFROUTE=no +PROXY_METHOD=none +BROWSER_ONLY=no +MTU=""" + neutron_network_mtu + """ +IPV4_FAILURE_FATAL=no +IPV6INIT=no +ETHTOOL_OPTS="-K ens4 tx-checksum-ip-generic off" +NAME="System ens4" +UUID=e27f182b-d125-2c43-5a30-43524d0229ac +""").encode() ifcfg_ens4_b64 = base64.standard_b64encode(ifcfg_ens4).decode().strip() config_data['ifcfg_ens4'] = {'base64': ifcfg_ens4_b64, 'path': '/etc/sysconfig/network-scripts/ifcfg-ens4'} opflex_conn = ("""VLAN=yes - TYPE=Vlan - PHYSDEV=""" + opflex_interface + """ - VLAN_ID=""" + infra_vlan + """ - REORDER_HDR=yes - GVRP=no - MVRP=no - PROXY_METHOD=none - BROWSER_ONLY=no - BOOTPROTO=dhcp - DEFROUTE=no - IPV4_FAILURE_FATAL=no - IPV6INIT=no - NAME=opflex-conn - DEVICE=""" + opflex_interface + """.""" + infra_vlan + """ - ONBOOT=yes - MTU=""" + neutron_network_mtu).encode() +TYPE=Vlan +PHYSDEV=""" + opflex_interface + """ +VLAN_ID=""" + infra_vlan + """ +REORDER_HDR=yes +GVRP=no +MVRP=no +PROXY_METHOD=none +BROWSER_ONLY=no +BOOTPROTO=dhcp +DEFROUTE=no +IPV4_FAILURE_FATAL=no +IPV6INIT=no +NAME=opflex-conn +DEVICE=""" + opflex_interface + """.""" + infra_vlan + """ +ONBOOT=yes +MTU=""" + neutron_network_mtu + """ +HWADDR= +ETHTOOL_OPTS="-K net0 tx-checksum-ip-generic off" +UUID=eb4377c5-a6d1-f09a-f588-7a6122be32f5 +""").encode() ifcfg_opflex_conn_b64 = base64.standard_b64encode(opflex_conn).decode().strip() config_data['ifcfg_opflex_conn'] = {'base64': ifcfg_opflex_conn_b64, 'path': '/etc/sysconfig/network-scripts/ifcfg-opflex-conn'} route_opflex_conn = """ADDRESS0=224.0.0.0 - NETMASK0=240.0.0.0 - METRIC0=1000""".encode() +NETMASK0=240.0.0.0 +METRIC0=1000 +""".encode() route_opflex_conn_b64 = base64.standard_b64encode(route_opflex_conn).decode().strip() @@ -218,10 +231,6 @@ def update(hostname,ignition): rendered_master = template_master.render(config_data) master_b64 = base64.standard_b64encode(rendered_master.encode()).decode().strip() - with open('./files/99_worker-disable-mco-validation-check.yaml', 'r') as f: - mc_config = f.read().encode() - mc_config_b64 = base64.standard_b64encode(mc_config).decode().strip() - files.append( { 'path': '/opt/openshift/openshift/99_master-networkscripts.yaml', @@ -244,17 +253,6 @@ def update(hostname,ignition): 'filesystem': 'root', }) - files.append( - { - 'path': '/opt/openshift/openshift/99_worker-disable-mco-validation-check.yaml', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + mc_config_b64, - 'verification': {} - }, - 'filesystem': 'root', - }) - hostname_b64 = base64.standard_b64encode(hostname).decode().strip() files.append( { From 0eb11e286c4b3d0c82c1d9f3ec6afcccc1d601ba Mon Sep 17 00:00:00 2001 From: ABHISHEK SHARMA Date: Tue, 2 Feb 2021 11:07:32 +0530 Subject: [PATCH 48/48] Removing the dependency of opflex interface from the bootstrap machine. 1. Bootstrap can run and function properly on single node interface and bring up the master nodes 2. Second opflex interface is not needed for it. --- upi/openstack/update_ign.py | 111 ++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 55 deletions(-) diff --git a/upi/openstack/update_ign.py b/upi/openstack/update_ign.py index fac37917305..db616fceb08 100644 --- a/upi/openstack/update_ign.py +++ b/upi/openstack/update_ign.py @@ -253,61 +253,62 @@ def update(hostname,ignition): 'filesystem': 'root', }) - hostname_b64 = base64.standard_b64encode(hostname).decode().strip() - files.append( - { - 'path': '/etc/hostname', - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + hostname_b64, - 'verification': {} - }, - 'filesystem': 'root', - }) - - files.append( - { - 'path': config_data['ifcfg_ens3']['path'], - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_ens3']['base64'], - 'verification': {} - }, - 'filesystem': 'root', - }) - - files.append( - { - 'path': config_data['ifcfg_ens4']['path'], - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_ens4']['base64'], - 'verification': {} - }, - 'filesystem': 'root', - }) - - files.append( - { - 'path': config_data['ifcfg_opflex_conn']['path'], - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_opflex_conn']['base64'], - 'verification': {} - }, - 'filesystem': 'root', - }) - - files.append( - { - 'path': config_data['route_opflex_conn']['path'], - 'mode': 420, - 'contents': { - 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['route_opflex_conn']['base64'], - 'verification': {} - }, - 'filesystem': 'root', - }) + else: + hostname_b64 = base64.standard_b64encode(hostname).decode().strip() + files.append( + { + 'path': '/etc/hostname', + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + hostname_b64, + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': config_data['ifcfg_ens3']['path'], + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_ens3']['base64'], + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': config_data['ifcfg_ens4']['path'], + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_ens4']['base64'], + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': config_data['ifcfg_opflex_conn']['path'], + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['ifcfg_opflex_conn']['base64'], + 'verification': {} + }, + 'filesystem': 'root', + }) + + files.append( + { + 'path': config_data['route_opflex_conn']['path'], + 'mode': 420, + 'contents': { + 'source': 'data:text/plain;charset=utf-8;base64,' + config_data['route_opflex_conn']['base64'], + 'verification': {} + }, + 'filesystem': 'root', + }) ignition['storage']['files'] = files return ignition