diff options
Diffstat (limited to 'resources/tools/testbed-setup/ansible')
55 files changed, 1117 insertions, 153 deletions
diff --git a/resources/tools/testbed-setup/ansible/cobbler.yaml b/resources/tools/testbed-setup/ansible/cobbler.yaml new file mode 100644 index 0000000000..06e6238e18 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/cobbler.yaml @@ -0,0 +1,10 @@ +--- +# file: cobbler.yaml + +- hosts: cobbler + remote_user: testuser + become: yes + become_user: root + roles: + - role: cobbler + tags: cobbler diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml index cae851dc93..f21ef477a5 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml @@ -1,13 +1,14 @@ --- -# file: group_vars/all.yaml +# file: lf_inventory/group_vars/all.yaml # General variables ansible_python_interpreter: '/usr/bin/python2.7' -ansible_become_pass: '{{ inventory_sudo_pass }}' +# provision via cobbler +provision_enabled: False # name_servers_search is used in /etc/hosts file on target machine. name_servers_search: 'linuxfoundation.org' # name_servers is used in /etc/netplan/01-netcfg.yaml -name_servers: "[ 199.204.44.24, 199.204.47.54 ]" +name_servers: "199.204.44.24, 199.204.47.54" # Proxy settings: Uncomment and fill the proper values. These variables will be # set globally by writing into /etc/environment file on target machine. diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.16.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.16.yaml index 3bf651e5e1..740c4755db 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.16.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.16.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_cimc_hostname: '10.30.50.16' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.17.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.17.yaml index 558cfd4cbf..0df2da30ed 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.17.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.17.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_cimc_hostname: '10.30.50.17' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.18.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.18.yaml index f33f7629b3..4d00c388c7 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.18.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.18.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_cimc_hostname: '10.30.50.18' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.20.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.20.yaml index 1ab5df17d1..688db252a1 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.20.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.20.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_cimc_hostname: '10.30.50.20' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.21.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.21.yaml index d3c552f384..79dd07493f 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.21.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.21.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_cimc_hostname: '10.30.50.21' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml index 1f6157f88d..69bc1178be 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.22.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_cimc_hostname: '10.30.50.22' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml index cd4b466c7d..f4db10ee44 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.24.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_cimc_hostname: '10.30.50.24' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml index 402944bc44..6a8c9f28b3 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.25.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_cimc_hostname: '10.30.50.25' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml index fdf70dffbe..d914eb17f8 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.26.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_cimc_hostname: '10.30.50.26' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml index 11665983fa..5e81b3573c 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml @@ -8,3 +8,6 @@ virl_l2_gateway: "10.30.52.1" virl_l2_network: "10.30.52.0/24" virl_l2_ip: "10.30.52.254" virl_public_port: "eth0" + +inventory_cimc_hostname: '10.30.50.28' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml index 4ab6fd1116..075a3920db 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml @@ -8,3 +8,6 @@ virl_l2_gateway: "10.30.53.1" virl_l2_network: "10.30.53.0/24" virl_l2_ip: "10.30.53.254" virl_public_port: "eth4" + +inventory_cimc_hostname: '10.30.50.29' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml index ab5d222188..6ed90238bc 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml @@ -8,3 +8,6 @@ virl_l2_gateway: "10.30.54.1" virl_l2_network: "10.30.54.0/24" virl_l2_ip: "10.30.54.254" virl_public_port: "eth0" + +inventory_cimc_hostname: '10.30.50.30' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.44.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.44.yaml index 4776829b29..85f4dd7d5b 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.44.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.44.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.41' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.45.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.45.yaml index 2715ea7118..be27ba59c1 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.45.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.45.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.42' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.46.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.46.yaml index 850b206dd7..74ad358587 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.46.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.46.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.43' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.47.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.47.yaml index 49da361833..f1313329a8 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.47.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.47.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.44' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.48.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.48.yaml index ef18a47baa..e0860e4d52 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.48.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.48.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.45' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.49.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.49.yaml index 64439733f5..23733be8d0 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.49.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.49.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.46' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.50.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.50.yaml index 87e336a958..36e2f8e840 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.50.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.50.yaml @@ -4,3 +4,6 @@ hostname: "s1-t11-sut1" grub: nr_hugepages: 65536 + +inventory_ipmi_hostname: '10.30.50.47' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.51.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.51.yaml index ff2f805484..472878a635 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.51.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.51.yaml @@ -4,3 +4,6 @@ hostname: "s2-t12-sut1" grub: nr_hugepages: 65536 + +inventory_ipmi_hostname: '10.30.50.48' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.52.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.52.yaml index 2bc3fdeb7a..a813ac6e90 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.52.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.52.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 65536 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.49' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.53.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.53.yaml index c9c51b8444..d85d56f51e 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.53.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.53.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.50' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.54.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.54.yaml index 877fab7806..d4676dda64 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.54.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.54.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.51' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.55.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.55.yaml index b5fa1ac685..61f61c7413 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.55.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.55.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.52' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.56.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.56.yaml index d1efda22e3..0fecacacb3 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.56.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.56.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.53' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.57.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.57.yaml index 162509deed..e56d03652b 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.57.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.57.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.54' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.58.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.58.yaml index cb21647678..03f364819a 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.58.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.58.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.55' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.59.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.59.yaml index b38bea10ab..b1aa02a1ec 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.59.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.59.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 36864 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.56' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.60.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.60.yaml index 106fba7dd1..ec62ec4060 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.60.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.60.yaml @@ -12,3 +12,6 @@ sysctl: vm: nr_hugepages: 4096 max_map_count: 20000 + +inventory_ipmi_hostname: '10.30.50.57' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts index d2d67981b0..35c4e2254f 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts @@ -15,6 +15,9 @@ all: 10.30.51.22: null #t2-sut2 10.30.51.25: null #t3-sut1 10.30.51.26: null #t3-sut2 + cobbler: + hosts: + 10.30.51.28: null #s4-virl1 skylake: children: tg: diff --git a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml index 8ffac218c3..a8e6d0f3a5 100644 --- a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml @@ -1,13 +1,14 @@ --- # file: group_vars/all.yaml -# General settings +# General variables ansible_python_interpreter: '/usr/bin/python2.7' -ansible_become_pass: '{{ inventory_sudo_pass }}' -# Domain is used in /etc/hosts file on target machine. -domain: 'linuxfoundation.org' -# DNS is used in /etc/netplan/01-netcfg.yaml on target machine. -dns_servers: "[ 199.204.44.24, 199.204.47.54 ]" +# provision via cobbler +provision_enabled: True +# name_servers_search is used in /etc/hosts file on target machine. +name_servers_search: 'linuxfoundation.org' +# name_servers is used in /etc/netplan/01-netcfg.yaml +name_servers: "199.204.44.24, 199.204.47.54" # Proxy settings: Uncomment and fill the proper values. These variables will be # set globally by writing into /etc/environment file on target machine. @@ -40,3 +41,15 @@ kubernetes_channel: 'main' kubernetes_version: '1.11.0-00' kubernetes_repository: 'deb http://apt.kubernetes.io/ kubernetes-xenial {{ kubernetes_channel }}' kubernetes_apt_package_name: '{{ kubernetes_version }}' + +# DPDK settings. +dpdk: + target_dir: '/opt' + version: 'dpdk-19.02' + url: 'https://fast.dpdk.org/rel' + +# Calibration settings. +jitter: + directory: '/tmp/pma_tools' + core: 7 + iterations: 30 diff --git a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/host_vars/1.1.1.1.yaml b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/host_vars/1.1.1.1.yaml index 89235e89c3..1cc3e5eaa2 100644 --- a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/host_vars/1.1.1.1.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/host_vars/1.1.1.1.yaml @@ -1,14 +1,17 @@ --- # file: host_vars/x.x.x.x.yaml -hostname: "t1-tg1" +hostname: 't1-tg1' grub: - - isolcpus: "1-27,29-55,57-83,85-111" - - nohz_full: "1-27,29-55,57-83,85-111" - - rcu_nocbs: "1-27,29-55,57-83,85-111" + - isolcpus: '1-27,29-55,57-83,85-111' + - nohz_full: '1-27,29-55,57-83,85-111' + - rcu_nocbs: '1-27,29-55,57-83,85-111' sysctl: - kernel: - - watchdog_cpumask: "0,28,56,84" + - watchdog_cpumask: '0,28,56,84' - vm: - nr_hugepages: 4096 - max_map_count: 20000 + +inventory_ipmi_hostname: 'x.x.x.x' +cobbler_profile: 'ubuntu-18.04.2-server-x86_64' diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/Dockerfile b/resources/tools/testbed-setup/ansible/roles/cobbler/files/Dockerfile new file mode 100644 index 0000000000..8d70c0b495 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/Dockerfile @@ -0,0 +1,96 @@ +# Copyright (c) 2019 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos:7 + +MAINTAINER csit-dev <csit-dev@lists.fd.io> +LABEL Description="CSIT cobbler service image." +LABEL Version="0.1" + +# Build arguments +ARG cobbler_sys_pass +ARG cobbler_web_pass +ARG cobbler_ip_addr +ARG cobbler_http_port=60080 +ARG cobbler_https_port=60443 + +# Install dependencies +RUN yum -y install epel-release \ + && yum -y install \ + cobbler \ + cobbler-web \ + fence-agents \ + python-pip \ + curl \ + dhcp \ + bind \ + file \ + debmirror \ + net-tools \ + rsync \ + pykickstart \ + supervisor \ + wget \ + which \ + && yum clean all \ + && rm -rf /var/cache/yum + +# Workaround for Cobbler 2.8.4 bug +RUN pip2.7 install -U django==1.9.13 + +# Copy CSIT configration +COPY supervisord/supervisord.conf /etc/supervisord.conf +COPY etc/cobbler/dhcp.template /etc/cobbler/dhcp.template +COPY var/lib/cobbler/kickstarts/* /var/lib/cobbler/kickstarts/ +COPY etc/httpd/conf.d/cobbler_web.conf /etc/httpd/conf.d/cobbler_web.conf + +RUN sed -i \ + -e "/^default_password_crypted/ s|:.*$|: \"${cobbler_sys_pass}\"|" \ + -e "/^next_server:/ s/:.*$/: ${cobbler_ip_addr}/" \ + -e "/^server/ s/:.*$/: ${cobbler_ip_addr}/" \ + -e "/^http_port:/ s/:.*$/: ${cobbler_http_port}/" \ + -e "/^pxe_just_once:/ s/:.*$/: 1/" \ + -e "/^manage_dhcp:/ s/:.*$/: 1/" \ + /etc/cobbler/settings \ + && sed -i "s/service %s restart/supervisorctl restart %s/g" \ + /usr/lib/python2.7/site-packages/cobbler/modules/sync_post_restart_services.py \ + && sed -i "s/Listen 80/Listen ${cobbler_http_port}/g" \ + /etc/httpd/conf/httpd.conf \ + && sed -i "s/Listen 443 https/Listen ${cobbler_https_port} https/g" \ + /etc/httpd/conf.d/ssl.conf + +# Change Cobbler WEB password +RUN echo -n "cobbler:Cobbler:${cobbler_web_pass}" \ + | md5sum \ + | cut -d' ' -f1 \ + | xargs printf "%s:%s:%s\n" cobbler Cobbler > "/etc/cobbler/users.digest" + +# Create Cobbler directories +RUN mkdir -p /var/lib/cobbler/config/distros.d \ + && mkdir -p /var/lib/cobbler/config/files.d \ + && mkdir -p /var/lib/cobbler/config/images.d \ + && mkdir -p /var/lib/cobbler/config/mgmtclasses.d \ + && mkdir -p /var/lib/cobbler/config/packages.d \ + && mkdir -p /var/lib/cobbler/config/profiles.d \ + && mkdir -p /var/lib/cobbler/config/repos.d \ + && mkdir -p /var/lib/cobbler/config/systems.d \ + && mkdir -p /var/www/cobbler/links/ \ + && touch /usr/share/cobbler/web/cobbler.wsgi + +# Expose TFTP WWW COBBLER +EXPOSE 69 +EXPOSE $cobbler_http_port +EXPOSE $cobbler_https_port +EXPOSE 25151 + +ENTRYPOINT /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template new file mode 100644 index 0000000000..cf2fbdfe34 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template @@ -0,0 +1,86 @@ +# ****************************************************************** +# Cobbler managed dhcpd.conf file +# +# generated from cobbler dhcp.conf template ($date) +# Do NOT make changes to /etc/dhcpd.conf. Instead, make your changes +# in /etc/cobbler/dhcp.template, as /etc/dhcpd.conf will be +# overwritten. +# +# ****************************************************************** + +ddns-update-style interim; + +allow booting; +allow bootp; + +ignore client-updates; +set vendorclass = option vendor-class-identifier; + +option pxe-system-type code 93 = unsigned integer 16; + +subnet 10.30.51.0 netmask 255.255.255.0 { + option routers 10.30.51.1; + option domain-name "linuxfoundation.org"; + option domain-name-servers 199.204.44.24, 199.204.47.54; + option subnet-mask 255.255.255.0; + range dynamic-bootp 10.30.51.2 10.30.51.254; + default-lease-time 600; + max-lease-time 7200; + next-server $next_server; + class "pxeclients" { + match if substring (option vendor-class-identifier, 0, 9) = "PXEClient"; + if option pxe-system-type = 00:02 { + filename "ia64/elilo.efi"; + } else if option pxe-system-type = 00:06 { + filename "grub/grub-x86.efi"; + } else if option pxe-system-type = 00:07 { + filename "grub/grub-x86_64.efi"; + } else { + filename "pxelinux.0"; + } + } +} + +#for dhcp_tag in $dhcp_tags.keys(): + ## group could be subnet if your dhcp tags line up with your subnets + ## or really any valid dhcpd.conf construct ... if you only use the + ## default dhcp tag in cobbler, the group block can be deleted for a + ## flat configuration +# group for Cobbler DHCP tag: $dhcp_tag +group { + #for mac in $dhcp_tags[$dhcp_tag].keys(): + #set iface = $dhcp_tags[$dhcp_tag][$mac] + host $iface.name { + hardware ethernet $mac; + #if $iface.ip_address: + fixed-address $iface.ip_address; + #end if + #if $iface.hostname: + option host-name "$iface.hostname"; + #end if + #if $iface.netmask: + option subnet-mask $iface.netmask; + #end if + #if $iface.gateway: + option routers $iface.gateway; + #end if + #if $iface.enable_gpxe: + if exists user-class and option user-class = "gPXE" { + filename "http://$cobbler_server/cblr/svc/op/gpxe/system/$iface.owner"; + } else if exists user-class and option user-class = "iPXE" { + filename "http://$cobbler_server/cblr/svc/op/gpxe/system/$iface.owner"; + } else { + filename "undionly.kpxe"; + } + #else + filename "$iface.filename"; + #end if + ## Cobbler defaults to $next_server, but some users + ## may like to use $iface.system.server for proxied setups + next-server $next_server; + ## next-server $iface.next_server; + } + #end for +} +#end for + diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/httpd/conf.d/cobbler_web.conf b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/httpd/conf.d/cobbler_web.conf new file mode 100644 index 0000000000..8b0f9863c1 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/httpd/conf.d/cobbler_web.conf @@ -0,0 +1,33 @@ +# This configuration file enables the cobbler web +# interface (django version) + +<Directory "/usr/share/cobbler/web/"> + SetEnv VIRTUALENV + Options Indexes MultiViews + AllowOverride None + Order allow,deny + Allow from all +</Directory> + +<Directory "/var/www/cobbler_webui_content/"> + Options +Indexes +FollowSymLinks + AllowOverride None + Order allow,deny + Allow from all +</Directory> + +# Use separate process group for wsgi +WSGISocketPrefix /var/run/wsgi +WSGIScriptAlias /cobbler_web /usr/share/cobbler/web/cobbler.wsgi +WSGIDaemonProcess cobbler_web display-name=%{GROUP} +WSGIProcessGroup cobbler_web +WSGIPassAuthorization On + +<IfVersion >= 2.4> + <Location /cobbler_web> + Require all granted + </Location> + <Location /cobbler_webui_content> + Require all granted + </Location> +</IfVersion> diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/supervisord/supervisord.conf b/resources/tools/testbed-setup/ansible/roles/cobbler/files/supervisord/supervisord.conf new file mode 100644 index 0000000000..4ac5af8f99 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/supervisord/supervisord.conf @@ -0,0 +1,42 @@ +[unix_http_server] +file=/run/supervisor.sock + +[supervisord] +pidfile=/var/run/supervisord.pid +identifier=supervisor +directory=/run +logfile=/var/log/supervisord.log +loglevel=debug +nodaemon=true + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/supervisor.sock + +[program:cobblerd] +command=/usr/bin/cobblerd -F +stdout_logfile=/var/log/supervisord.log +stderr_logfile=/var/log/supervisord.log + +[program:httpd] +command=/usr/sbin/httpd -DFOREGROUND +stdout_logfile=/var/log/supervisord.log +stderr_logfile=/var/log/supervisord.log + +[program:tftpd] +command=/usr/sbin/in.tftpd --foreground --verbose --user root --permissive --blocksize 1380 --address 0.0.0.0:69 --secure /var/lib/tftpboot +stdout_logfile=/var/log/supervisord.log +stderr_logfile=/var/log/supervisord.log + +[program:rsyncd] +command=/usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf +stopsignal=QUIT +stdout_logfile=/var/log/supervisord.log +stderr_logfile=/var/log/supervisord.log + +[program:dhcpd] +command=/usr/sbin/dhcpd -f -cf /etc/dhcp/dhcpd.conf -user dhcpd -group dhcpd --no-pid +stdout_logfile=/var/log/supervisord.log +stderr_logfile=/var/log/supervisord.log diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed new file mode 100644 index 0000000000..250701232c --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed @@ -0,0 +1,137 @@ +#### Contents of the preconfiguration file (for Ubuntu 18.04 Server LTS) + +### Localization +# Locale sets language and country. +d-i debian-installer/locale string en_US +d-i pkgsel/install-language-support boolean false +# Keyboard selection. +# Disable automatic (interactive) keymap detection. +d-i console-setup/ask_detect boolean false +d-i console-setup/layoutcode string us + +### Network configuration +# netcfg will choose an interface that has link if possible. This makes it +# skip displaying a list if there is more than one interface. +d-i netcfg/choose_interface select auto +# If you have a slow dhcp server and the installer times out waiting for +# it, this might be useful. +d-i netcfg/dhcp_timeout string 60 +# If you prefer to configure the network manually, uncomment this line and +# the static network configuration below. +#d-i netcfg/disable_dhcp boolean true +# If you want the preconfiguration file to work on systems both with and +# without a dhcp server, uncomment these lines and the static network +# configuration below. +d-i netcfg/dhcp_failed note +#d-i netcfg/dhcp_options select Configure network manually +# Static network configuration. +#d-i netcfg/get_nameservers string 172.30.0.2 +#d-i netcfg/get_ipaddress string 172.30.15.42 +#d-i netcfg/get_netmask string 255.255.0.0 +#d-i netcfg/get_gateway string 172.30.0.1 +#d-i netcfg/confirm_static boolean true +# Any hostname and domain names assigned from dhcp take precedence over +# values set here. However, setting the values still prevents the questions +# from being shown, even if values come from dhcp. +d-i netcfg/get_hostname string unassigned-hostname +d-i netcfg/get_domain string unassigned-domain +# Disable WEP key dialog. +d-i netcfg/wireless_wep string +# Alternatively: by default, the installer uses CC.archive.ubuntu.com where +# CC is the ISO-3166-2 code for the selected country. You can preseed this +# so that it does so without asking. +d-i mirror/http/mirror select us.archive.ubuntu.com + +### Mirror settings +d-i live-installer/net-image string http://$http_server/cobbler/links/$distro_name/install/filesystem.squashfs + +### Clock and time zone setup +# Controls whether or not the hardware clock is set to UTC. +d-i clock-setup/utc boolean true +# You may set this to any valid setting for $TZ; see the contents of +# /usr/share/zoneinfo/ for valid values. +d-i time/zone string America/Los_Angeles +# Controls whether to use NTP to set the clock during the install +d-i clock-setup/ntp boolean false + +### Partitioning +## If the system has free space you can choose to only partition that space. +# Alternatives: custom, some_device, some_device_crypto, some_device_lvm. +d-i partman-auto/init_automatically_partition select some_device +# Alternatively, you can specify a disk to partition. The device name must +# be given in traditional non-devfs format. +d-i partman-auto/disk string /dev/sda +# In addition, you'll need to specify the method to use. +# The presently available methods are: "regular", "lvm" and "crypto" +d-i partman-auto/method string regular +# If one of the disks that are going to be automatically partitioned +# contains an old LVM configuration, the user will normally receive a +# warning. This can be preseeded away... +d-i partman-lvm/device_remove_lvm boolean true +# The same applies to pre-existing software RAID array: +d-i partman-md/device_remove_md boolean true +# And the same goes for the confirmation to write the lvm partitions. +d-i partman-lvm/confirm boolean true +# You can choose one of the three predefined partitioning recipes: +# - atomic: all files in one partition +# - home: separate /home partition +# - multi: separate /home, /usr, /var, and /tmp partitions +d-i partman-auto/choose_recipe select atomic +# If you just want to change the default filesystem from ext3 to something +# else, you can do that without providing a full recipe. +d-i partman/default_filesystem string ext4 +# This makes partman automatically partition without confirmation, provided +# that you told it what to do using one of the methods above. +d-i partman/confirm_write_new_label boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true + +### Account setup +# Skip creation of a root account (normal user account will be able to +# use sudo). The default is false; preseed this to true if you want to set +# a root password. +d-i passwd/root-login boolean false +# Root password, either in clear text +#d-i passwd/root-password password pass +#d-i passwd/root-password-again password pass +# or encrypted using an MD5 hash. +#d-i passwd/root-password-crypted password f4f1d7b6738330f521de21da3f563bce +# To create a normal user account. +d-i passwd/user-fullname string testuser +d-i passwd/username string testuser +# Normal user's password, either in clear text +d-i passwd/user-password password Csit1234 +d-i passwd/user-password-again password Csit1234 +# or encrypted using an MD5 hash. +#d-i passwd/user-password-crypted password f4f1d7b6738330f521de21da3f563bce +# The installer will warn about weak passwords. If you are sure you know +# what you're doing and want to override it, uncomment this. +d-i user-setup/allow-password-weak boolean true +# Set to true if you want to encrypt the first user's home directory. +d-i user-setup/encrypt-home boolean false + +### Package selection +tasksel tasksel/first multiselect ubuntu-server +# Individual additional packages to install +d-i pkgsel/include string openssh-server python2.7 +# Whether to upgrade packages after debootstrap. +# Allowed values: none, safe-upgrade, full-upgrade +#d-i pkgsel/upgrade select none +# Policy for applying updates. May be "none" (no automatic updates), +# "unattended-upgrades" (install security updates automatically), or +# "landscape" (manage system with Landscape). +d-i pkgsel/update-policy select none + +### Boot loader installation +d-i grub-installer/bootdev string default +# This is fairly safe to set, it makes grub install automatically to the MBR +# if no other operating system is detected on the machine. +d-i grub-installer/only_debian boolean false +# This one makes grub-installer install to the MBR if it also finds some other +# OS, which is less safe as it might not be able to boot that other OS. +d-i grub-installer/with_other_os boolean false + +### Finishing up the installation +# Avoid that last message about the install being complete. +d-i finish-install/reboot_in_progress note diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/cobbler/tasks/main.yaml new file mode 100644 index 0000000000..f48a976ea1 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cobbler/tasks/main.yaml @@ -0,0 +1,45 @@ +--- +# file: roles/cobbler/tasks/main.yaml + +- name: Sync the cobbler docker directory + synchronize: + src: 'files' + dest: '/home/{{ ansible_user }}/cobbler_docker' + register: __cobbler_image_built + tags: cobbler-build-image + +- name: Build the cobbler docker image + docker_image: + path: '/home/{{ ansible_user }}/cobbler_docker/files' + name: 'csit/cobbler' + buildargs: + cobbler_pass: '{{ cobbler_pass }}' + cobbler_web_pass: '{{ cobbler_password }}' + cobbler_ip_addr: '{{ inventory_hostname }}' + when: __cobbler_image_built + tags: cobbler-build-image + +- name: Run Cobbler image + docker_container: + name: 'cobbler' + image: 'csit/cobbler' + network_mode: 'host' + volumes: + - '/mnt:/mnt:ro' + register: __cobbler_image_running + tags: cobbler-run-image + +- name: Run cobbler setup get-loaders + command: 'docker exec -i cobbler cobbler get-loaders' + when: __cobbler_image_running + tags: cobbler-run-image + +- name: Run cobbler setup sync + command: 'docker exec -i cobbler cobbler sync' + when: __cobbler_image_running + tags: cobbler-run-image + +- name: Add Ubuntu 18.04.2 Server x86_64 to cobbler + include_tasks: 'ubuntu-18.04.2-server-x86_64.yaml' + when: __cobbler_image_running + tags: cobbler-run-image diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/tasks/ubuntu-18.04.2-server-x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/cobbler/tasks/ubuntu-18.04.2-server-x86_64.yaml new file mode 100644 index 0000000000..2c89234de2 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/cobbler/tasks/ubuntu-18.04.2-server-x86_64.yaml @@ -0,0 +1,35 @@ +--- +# file: roles/cobbler/tasks/ubuntu-18.04.2-server-x86_64.yaml + +- name: Download Ubuntu 18.04.2 Server x86_64 with check (sha256) + get_url: + url: 'http://cdimage.ubuntu.com/ubuntu/releases/18.04/release/ubuntu-18.04.2-server-amd64.iso' + dest: '/mnt/ubuntu-18.04.2-server-amd64.iso' + checksum: 'sha256:a2cb36dc010d98ad9253ea5ad5a07fd6b409e3412c48f1860536970b073c98f5' + register: __iso_downloaded + tags: cobbler-import-image + +- name: Create directory for Ubuntu 18.04.2 Server x86_64 mount + file: + path: '/mnt/ubuntu-18.04.2-server-x86_64' + state: 'directory' + register: __mount_directory_created + tags: cobbler-import-image + +- name: Mount Ubuntu 18.04.2 Server x86_64 iso + mount: + src: '/mnt/ubuntu-18.04.2-server-amd64.iso' + path: '/mnt/ubuntu-18.04.2-server-x86_64' + fstype: 'iso9660' + opts: 'ro,loop' + state: mounted + when: __iso_downloaded and __mount_directory_created + tags: cobbler-import-image + +- name: Run cobbler distro import for Ubuntu 18.04.2 Server x86_64 + command: | + docker exec -i cobbler cobbler import + --path=/mnt/ubuntu-18.04.2-server-x86_64 + --name=ubuntu-18.04.2-server-x86_64 + --kickstart=/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed + tags: cobbler-import-image diff --git a/resources/tools/testbed-setup/ansible/roles/common/handlers/cimc.yaml b/resources/tools/testbed-setup/ansible/roles/common/handlers/cimc.yaml new file mode 100644 index 0000000000..e31d712d4d --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/common/handlers/cimc.yaml @@ -0,0 +1,69 @@ +--- +# file: roles/common/handlers/cimc.yaml + +- name: Boot from network + imc_rest: + hostname: '{{ inventory_cimc_hostname }}' + username: '{{ inventory_cimc_username }}' + password: '{{ inventory_cimc_password }}' + validate_certs: no + content: | + <!-- Configure PXE boot --> + <configConfMo><inConfig> + <lsbootLan dn="sys/rack-unit-1/boot-policy/lan-read-only" access="read-only" order="1" prot="pxe" type="lan"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: boot-network + +- name: Boot from storage + imc_rest: + hostname: '{{ inventory_cimc_hostname }}' + username: '{{ inventory_cimc_username }}' + password: '{{ inventory_cimc_password }}' + validate_certs: no + content: | + <configConfMo><inConfig> + <lsbootStorage dn="sys/rack-unit-1/boot-policy/storage-read-write" access="read-write" order="1" type="storage"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: boot-storage + +- name: Power up server + imc_rest: + hostname: '{{ inventory_cimc_hostname }}' + username: '{{ inventory_cimc_username }}' + password: '{{ inventory_cimc_password }}' + validate_certs: no + content: | + <configConfMo><inConfig> + <computeRackUnit dn="sys/rack-unit-1" adminPower="up"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: power-up + +- name: Power down server + imc_rest: + hostname: '{{ inventory_cimc_hostname }}' + username: '{{ inventory_cimc_username }}' + password: '{{ inventory_cimc_password }}' + validate_certs: no + content: | + <configConfMo><inConfig> + <computeRackUnit dn="sys/rack-unit-1" adminPower="down"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: power-down + +- name: Power cycle server + imc_rest: + hostname: '{{ inventory_cimc_hostname }}' + username: '{{ inventory_cimc_username }}' + password: '{{ inventory_cimc_password }}' + validate_certs: no + content: | + <!-- Power cycle server --> + <configConfMo><inConfig> + <computeRackUnit dn="sys/rack-unit-1" adminPower="cycle-immediate"/> + </inConfig></configConfMo> + delegate_to: localhost + tags: power-cycle diff --git a/resources/tools/testbed-setup/ansible/roles/common/handlers/ipmi.yaml b/resources/tools/testbed-setup/ansible/roles/common/handlers/ipmi.yaml new file mode 100644 index 0000000000..6252cc04fd --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/common/handlers/ipmi.yaml @@ -0,0 +1,47 @@ +--- +# file: roles/common/handlers/ipmi.yaml + +- name: Boot from network + ipmi_boot: + name: '{{ inventory_ipmi_hostname }}' + user: '{{ inventory_ipmi_username }}' + password: '{{ inventory_ipmi_password }}' + bootdev: network + delegate_to: localhost + tags: boot-network + +- name: Boot from storage + ipmi_boot: + name: '{{ inventory_ipmi_hostname }}' + user: '{{ inventory_ipmi_username }}' + password: '{{ inventory_ipmi_password }}' + bootdev: hd + delegate_to: localhost + tags: boot-storage + +- name: Power up server + ipmi_power: + name: '{{ inventory_ipmi_hostname }}' + user: '{{ inventory_ipmi_username }}' + password: '{{ inventory_ipmi_password }}' + state: on + delegate_to: localhost + tags: power-up + +- name: Power down server + ipmi_power: + name: '{{ inventory_ipmi_hostname }}' + user: '{{ inventory_ipmi_username }}' + password: '{{ inventory_ipmi_password }}' + state: off + delegate_to: localhost + tags: power-down + +- name: Power cycle server + ipmi_power: + name: '{{ inventory_ipmi_hostname }}' + user: '{{ inventory_ipmi_username }}' + password: '{{ inventory_ipmi_password }}' + state: boot + delegate_to: localhost + tags: power-cycle diff --git a/resources/tools/testbed-setup/ansible/roles/common/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/handlers/main.yaml index bdcf6832f4..f994afdc60 100644 --- a/resources/tools/testbed-setup/ansible/roles/common/handlers/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/common/handlers/main.yaml @@ -1,6 +1,30 @@ --- # file: roles/common/handlers/main.yaml +- name: IPMI specific + import_tasks: ipmi.yaml + when: inventory_ipmi_hostname is defined + tags: ipmi-handlers + +- name: CIMC specific + import_tasks: cimc.yaml + when: inventory_cimc_hostname is defined + tags: cimc-handlers + - name: Update GRUB command: update-grub tags: update-grub + +- name: Reboot server + reboot: + reboot_timeout: 3600 + tags: reboot-server + +- name: Wait for server to restart + wait_for: + host: '{{ inventory_hostname }}' + search_regex: OpenSSH + port: 22 + delay: 60 + timeout: 3600 + tags: reboot-server diff --git a/resources/tools/testbed-setup/ansible/roles/common/handlers/reboot.yaml b/resources/tools/testbed-setup/ansible/roles/common/handlers/reboot.yaml deleted file mode 100644 index 01e1eb1b6b..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/common/handlers/reboot.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# file roles/common/handlers/reboot.yaml - -- name: Reboot host - command: shutdown -r now "Ansible updates triggered" - async: 0 - poll: 0 - ignore_errors: true - tags: reboot-host - -- name: Waiting for server to come back - local_action: wait_for host={{ inventory_hostname }} - state=started - tags: reboot-host diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml index d7355883e0..43e83105ce 100644 --- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml @@ -1,29 +1,62 @@ --- # file: roles/common/tasks/main.yaml +- name: Ensure the system exists in Cobbler + cobbler_system: + host: '{{ cobbler_hostname }}' + port: 60080 + interfaces: + br1: + ipaddress: '{{ ansible_default_ipv4.address }}' + macaddress: '{{ ansible_default_ipv4.macaddress }}' + name: '{{ hostname }}' + password: '{{ cobbler_password }}' + properties: + hostname: '{{ hostname }}' + gateway: '{{ ansible_default_ipv4.gateway }}' + profile: '{{ cobbler_profile }}' + name_servers: '{{ name_servers }}' + name_servers_search: '{{ name_servers_search }}' + kickstart: '/var/lib/cobbler/kickstarts/{{ cobbler_profile }}.seed' + kernel_options: '"interface={{ ansible_default_ipv4.interface }}"' + netboot_enabled: yes + username: '{{ cobbler_username }}' + use_ssl: no + validate_certs: no + when: provision_enabled + delegate_to: localhost + tags: cobbler-include + +- name: Commit Cobbler changes + cobbler_sync: + host: '{{ cobbler_hostname }}' + port: 60080 + password: '{{ cobbler_password }}' + username: '{{ cobbler_username }}' + use_ssl: no + validate_certs: no + run_once: yes + when: provision_enabled + delegate_to: localhost + register: __included_in_cobbler + notify: + - 'Boot from network' + - 'Reboot server' + tags: cobbler-include + +- meta: flush_handlers + - name: Add permanent proxy settings lineinfile: - path: "/etc/environment" - state: "present" - line: "{{ item.key }}={{ item.value }}" - with_dict: "{{ proxy_env }}" + path: '/etc/environment' + state: 'present' + line: '{{ item.key }}={{ item.value }}' + with_dict: '{{ proxy_env }}' when: proxy_env is defined - name: Install distribution - release - machine prerequisites include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' -- name: Set hostname - hostname: - name: '{{ hostname }}' - tags: set-hostname - -- name: Ensure hostname is in /etc/hosts - lineinfile: - path: '/etc/hosts' - regexp: '^{{ ansible_default_ipv4.address }}.+$' - line: '{{ ansible_default_ipv4.address }} {{ hostname }}.{{ name_servers_search }}' - tags: set-hostname - - name: Set sudoers admin lineinfile: path: '/etc/sudoers' @@ -49,13 +82,9 @@ owner: 'root' group: 'root' mode: '644' - notify: ['Update GRUB'] + notify: + - 'Update GRUB' + - 'Reboot server' tags: copy-grub -- name: Add permanent proxy settings - lineinfile: - path: "/etc/environment" - state: "present" - line: "{{ item.key }}={{ item.value }}" - with_dict: "{{ proxy_env }}" - when: proxy_env is defined +- meta: flush_handlers diff --git a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml index 506cb3ace5..962c333276 100644 --- a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml +++ b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml @@ -20,6 +20,9 @@ - 'python-cffi-backend' - 'python3-cffi' - 'python3-cffi-backend' + - 'libnuma-dev' + - 'lzop' + - 'lrzip' state: 'present' install_recommends: False tags: install-csit-dependencies diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/netplan_config b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/netplan_config index 7a6ae9e468..694bff1f4f 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/netplan_config +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/netplan_config @@ -9,4 +9,4 @@ network: dhcp4: false gateway4: {{ ansible_default_ipv4["gateway"] }} nameservers: - addresses: {{ name_servers }} + addresses: [ {{ name_servers }} ] diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/handlers/main.yaml index 15a6803671..8fbcb29de0 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/handlers/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/handlers/main.yaml @@ -6,3 +6,17 @@ name: 'docker' state: 'restarted' tags: restart-docker + +- name: Reboot server + reboot: + reboot_timeout: 3600 + tags: reboot-server + +- name: Wait for server to restart + wait_for: + host: '{{ inventory_hostname }}' + search_regex: OpenSSH + port: 22 + delay: 60 + timeout: 3600 + tags: reboot-server diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml index 5c205b5a70..67c6c17220 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml @@ -15,6 +15,8 @@ - 'ca-certificates' - 'software-properties-common' - 'libnuma-dev' + - 'lzop' + - 'lrzip' state: 'present' install_recommends: False tags: install-csit-dependencies diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml index e5a2ad3c7a..dc0b685a7f 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/x86_64.yaml @@ -1,18 +1,22 @@ --- # file: roles/tg_sut/tasks/x86_64.yaml +- name: Load msr by default + lineinfile: + path: '/etc/modules' + state: 'present' + line: 'msr' + tags: disable-turbo-boost + - name: Configure x86_64 kernel parameters lineinfile: path: '/etc/default/grub' state: 'present' regexp: '^GRUB_CMDLINE_LINUX=' line: 'GRUB_CMDLINE_LINUX="isolcpus={{ grub.isolcpus }} nohz_full={{ grub.nohz_full }} rcu_nocbs={{ grub.rcu_nocbs }} numa_balancing=disable intel_pstate=disable intel_iommu=on iommu=pt nmi_watchdog=0 audit=0 nosoftlockup processor.max_cstate=1 intel_idle.max_cstate=1 hpet=disable tsc=reliable mce=off"' - notify: ['Update GRUB'] + notify: + - 'Update GRUB' + - 'Reboot server' tags: set-grub -- name: Load msr by default - lineinfile: - path: '/etc/modules' - state: 'present' - line: 'msr' - tags: disable-turbo-boost +- meta: flush_handlers diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/vpp_device/handlers/main.yaml index bda9dd4649..7b53dda7ab 100644 --- a/resources/tools/testbed-setup/ansible/roles/vpp_device/handlers/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/handlers/main.yaml @@ -1,11 +1,27 @@ --- # file: roles/vpp_device/handlers/main.yaml -- name: Enable csit-initialize-vfs.service - command: systemctl enable csit-initialize-vfs.service - tags: enable-vf-service +- name: Start csit-initialize-vfs.service + systemd: + enabled: yes + state: started + name: csit-initialize-vfs.service + tags: start-vf-service - name: Update GRUB command: update-grub tags: update-grub +- name: Reboot server + reboot: + reboot_timeout: 3600 + tags: reboot-server + +- name: Wait for server to restart + wait_for: + host: '{{ inventory_hostname }}' + search_regex: OpenSSH + port: 22 + delay: 60 + timeout: 3600 + tags: reboot-server diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml index c3fbfeb1c6..97dec04d8f 100644 --- a/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml @@ -10,21 +10,28 @@ mode: '744' tags: copy-vf-script -- name: Copy csit-initialize-vfs.service +- name: Start csit-initialize-vfs.service copy: src: 'files/csit-initialize-vfs.service' dest: '/etc/systemd/system/' owner: 'root' group: 'root' mode: '644' - notify: ['Enable csit-initialize-vfs.service'] - tags: copy-vf-service + notify: + - 'Start csit-initialize-vfs.service' + tags: start-vf-service -- name: Set isolcpus and pstate parameter +- meta: flush_handlers + +- name: Set hugepages in GRUB lineinfile: path: '/etc/default/grub' state: 'present' regexp: '^GRUB_CMDLINE_LINUX=' line: 'GRUB_CMDLINE_LINUX="hugepagesz=2M hugepages={{ grub.nr_hugepages }}"' - notify: ['Update GRUB'] + notify: + - 'Update GRUB' + - 'Reboot server' tags: set-grub + +- meta: flush_handlers diff --git a/resources/tools/testbed-setup/ansible/site.yaml b/resources/tools/testbed-setup/ansible/site.yaml index f2f68c6bb7..0d1913f8f6 100644 --- a/resources/tools/testbed-setup/ansible/site.yaml +++ b/resources/tools/testbed-setup/ansible/site.yaml @@ -9,3 +9,6 @@ - import_playbook: vpp_device.yaml tags: vpp-device + +- import_playbook: cobbler.yaml + tags: cobbler diff --git a/resources/tools/testbed-setup/ansible/vault.yml b/resources/tools/testbed-setup/ansible/vault.yml index 8f75700a09..a500462147 100644 --- a/resources/tools/testbed-setup/ansible/vault.yml +++ b/resources/tools/testbed-setup/ansible/vault.yml @@ -1,86 +1,255 @@ $ANSIBLE_VAULT;1.1;AES256 -30376536326632653166363236653465343033356264353839636231396564336338613335393164 -6666663765663664656665393034336538333731393863640a343337343739313266393063373637 -34383438656166303961373038373465323939333337666266656634326633393136333764643061 -3839623938643064320a386431376632313933666638306261333161396535343936393536326430 -38393835303931383166633136643531666338383661343833356237376265613461356165326262 -65346232386434393137623463623239323430326338633563646461383938653437613634336632 -38616337623662623930373237633936306231386337353536333238366133643061376463373437 -38616132643561613663383533666332623334636461393535313930316461356430303238306332 -38376533663062666438363639346436366266643532356236363738653432623536326266623633 -62633135386138393737373365656465316433623934656266333534346430316265613064646633 -34393566326332396564656234636231393330353639386337326131326665643865386466386331 -62613938363730373132323934666661383061613136346261353934303830623535333032623731 -62316163646231623666393665313939656366346432623631396661636535303762633062626331 -32643139343836623035623534333865313435393961383866303433373831653731306638653237 -38396533646330323838653162363435633030613334353965643132663632653837613266323739 -37313235623830656436326561303935623665383534303436636663653439643835663139343239 -64653662373837306330336662623766383135356232383238303664663235346231353939396331 -61326565333661653262656166643033653235636338643734333363336332333332636531313235 -61343161613366663637626462376334323032313463316661363766623161663832376661376363 -31326639386566663533346334383131326530323263626636646237386334613263323839386564 -39303732366362303162363330363238363030396261313566646530626362333261663966303034 -66333930333866303962356331303238306631336530383466633030313734323534613266633938 -61373763656663383230343563613537376363396331326434303032356635346231306636353132 -65373038333636373362336230386534386661333332373535633762323566623038626261663933 -32663031336165633333646438613666316562323263353731313735343234313234316261333335 -35326532376662373663366537333736373030356435376231393566663236376231633261393765 -33643166326234373436316330343264353032373030623736323162653062306232396265313234 -63363438393631376539653935326232626561346265633161326135616437613163643139333831 -65623962393939316533396664313932303730646138323761356266653861323635373166633061 -62323133316136366333386131343962346165376362376365313936353436646434313265333432 -33613432663666643361343161623731646561616464333530613164656566633931346665346537 -39313239653538656137333931373633636139633338333364373239366463353733323732373062 -39353938616139323463393730643733633338363763383265373264363136353335353431663537 -35326466396137663965326564353739356437366464376332633036303061663436623133396131 -37663965386136373532666632363862616337636633653264343331636363303166653733663834 -66656133653363303361636361616134306263383833346361346662353466363133303935306632 -33353537353637633262353366346534636662663531373232356536633735316565663432646361 -63383635376263646164363364666633373734613063363366376365393632623835383432333235 -30643136633039626239666163316434353333303739333761656365383764656132316537343966 -30643764333538323866313538626435663665663232323934333032653464663530633364316261 -30313161323839383662333036316562633937386636643462373439376165646432393765653539 -34303162316136643639643064363636383439616238626264383563313633326637333936373932 -38336435303663636466396637313061356261353431346361373037613737393364306234653038 -35663734643065336362373562313262323330396365613566633038643632633232366638323335 -39303933623837323163356430663330323433363038373530376439303933663434363365636338 -64653366626138656535303766323735626331303731626439343766313732643336363735353438 -37643433326464353661613664376639633631393462386131316663323935356535626164626363 -34623061663837636362363862616564353631633162396637393739643933306134626234656131 -66626464636636366663333238636135343938343333626238656339376239306261313937343666 -31353764326662666639623637366533623463393030636233626661313132393363316564343664 -37303866366531373136303731386638396562623837323666333238353732313162623537356636 -31396366653539336564363761663532653664663263653438303932346566373066353865373430 -65643264326635613762666465313638363132343537623034326465323937346463613263636231 -66653835616633383134656434326665373536663630316237343664396565383930373863366339 -62643961393534636161653832363736313039626330323766326238653861373735363339333636 -63616433343664323366666338376233366637313363633263353564326339326465666233353564 -31626664363061636437356332646238396331303739643637373533626261646531333534393232 -63613461356530613064343431336365653332366635666339656631306235616561613030363664 -39333438313865393632393535633865346431633139623565376363613933313536333335383030 -38633165346462383337313562616631323866636566373833626335363433343430646165383236 -32333266313065313764313062313162616536626266323061643733356164373062623533643432 -37353332343161666136313963613564626265633664356366383330643562363564396134383038 -32396362346666643833663266353131643736333333353430323035363131353663643365363233 -35303932383435396131663566383233623033633965363433356239316634653031333835353563 -38626564386531613135386161346439666134623134343632363234646638376233366136653636 -32373566373666613333323065626465653562656336336532383930633938303564353732373831 -64646631333535313562393634303464303932316632663463356663316538306666316335643631 -66366666633662623362396336386239356134363535666130666666636562643235363462303536 -36323364353735623266356537393732376438353531623930326664376661333933353436306339 -32386334623436343439343930336363366338623832343633653631633435353233356330366662 -37396139646364643934363331323035663830383864666538313965353463353362313631396537 -65353462333033633534376232653961306339363064376432326335353561323162353163356538 -31303232313739343331656164633332393337613433326364316432303733663663643163396664 -38616337356361616162353237373635653335326464336339653635356533326666666165616635 -62326432383565386266333633626130363063323630663635343363343261633837376436393936 -66366161386535316265613566313164656262343866666531363334366361663264653539343036 -65393061373538356637343733613036303435306565643338353930376565356665633865383339 -65326263323031363332356230373964373364623861353662363430656339383664613961373566 -66393733656635366266396665316164383464303438326232663463316532363631333538376364 -33336164373035353665333833306564633636653132393235323730333132313435316339303132 -63636663333038363338333562613032383135353033323964333336386666613337363737366230 -34646461323766323431663762666464613538663933396639316132653865373763633230626436 -35303365336663663236613832373137353333323534653031643934626232323530356136626461 -34373533313631626562653661616162333631336235323136366237633462666335376433323431 -6337 +36343463316430383361393130343966303537663633623764306562626538623763633630326565 +3739396238616663666564643633656563646262656261370a373361326534313433643737623761 +66643663633865373637323138626432613065376430303233623263323463633764373233333836 +6561643461303634360a346634346162633337366530333037633235633932643833643165626361 +38666564373132366635663264393632303265303633366238363735393335373932666665383164 +37623239326234323738613338346162613030326237653930646162326465653837653538366438 +62393734623663306465333435663133393562656566386334303133333733376433353431616561 +62386439333065393365346462333331633537313962363965353732313662376463626161323338 +34366663386631626137363131643732646530323863646536613166653136343862333236323837 +66323839636264346231643737336134633466396363343537333231626433376632383131356332 +62313665373639643139333537396463383962386439643530616633633238393430346231323665 +33343738616331303563353438653662303762306237663162383430386438333431343333383739 +32343265643534396362373635383638303135663063376638343330353662613365393237663738 +65323932303030373862653539626431316135363639623930366337366333623761356337303836 +33373561386563346231366134616130633939626334343366666233376430326533343133323937 +30393963363866393863653237343762316663646633313934666236333061326333613037353363 +61663334623331653932316361373634626432366139643938613435396436353562623234336365 +65313239363130656532303432306232636438386162646233666662373938353461363361393831 +61653264656133366231346235663132303930616566336630626530383538616637626139326537 +37393938623764633037383931323439306561363931316133363764323432343862353836383864 +35646263626531336237393133356264333135353731666337653765623863643637643735373063 +35653637353230636630636232626534666232613639353836363235303163366362646162633330 +32623564333030613566353565376130633365316266323836616631326363626162643438356366 +32623939623830663039373032396636383739356530346537653730613762633731623863376233 +36653632356331656530613865346365353363633466643731633736363533303732303930396336 +33356366323765643838393634373664626662366530646661363764333833366262333531616635 +31313239376535626434333431303538656662303162326235336138333635313132326438653361 +62656635323462656536336464316436373039323461306461353534613534633532303930623661 +35663961393339363661633932376639326539333536373637303535333562373233643730663565 +38653064336232396136306566353564633561613330666331626430626161316639303234346564 +39333732333964373861663261613130326164303330353039373132326332383965643737383566 +65356631313562376634393534396632663566616366346434396663323165623261626332323034 +62656436626236343736613664636264383735616331313730316533336366326635303936353535 +62323937326636326264396133313933616664633765373535666161346532616237626432333431 +36393463383736353663326366306161636230623038626138393162373231323361633061633134 +38323937376363313664336638656162333538313036393337306261383664353461653762366333 +36643435646465616265313638653537303664646332653332393430346363313435393833373536 +34643137353834386533373736373165623435396463393236333231363163316463653439373535 +34396437326331336435623464373466643164653063343435636535386461653937623730636234 +30663435396261343464663835313832306366323266356434653838303461633036666161376339 +32353435376562623436393662643233373333383862313635656331656366383664663661633764 +36383236633437636136366263386330316438346137313034316439666237336438383130653730 +63666231666234343936323439326538376330343865366234303961393435326530303638336330 +33343734333039303637646261396462346565363032323437656263656665373937623232343639 +61393263326463363363313439346630336536356332346662313334373936386261613830613361 +65356434373262653632363466313562386538396638393330393963613064663833656565323164 +31373761383362613365626563343935386462656562613533303637653662376665393731666366 +66386264306330356563343565656166666434346466623239623636393934653435306534356339 +33383533363938336164363133616261646638346336353862353362353439333539336436653165 +39303833353036646337333836633963396234323439396363316238313130333537643737303236 +65356263363566663639653533383737376530353035343562633032323635616565633763643035 +34623966643239633866323331666233653438633064316562353963643335653363383830626238 +37313137613837626639376238376562656463623834383633303830353861653736346539336665 +32656236343138356564356264626130383334303939666564383237666662613163636432383835 +37363463656333613464393438613633656165633166343634386534616333396462633337343765 +39343133386162326162653231636463323231653733636264336539386131353234363466303363 +61343465333537363038306138623766616130666534313733613033303530356364393265653136 +66663231313936656331643561363339383365623263363936303862653564643434326364373130 +37666664663663663530396265383965653639386430313938626665623933306666353631313538 +31353830363730626632376161663263623336623731613466323163663565626435303034326232 +66333161366563393537336264633935636465396239303865373233623938616564653030363066 +32326138343931613032323837653034666333353365643337326363393932646437316366653832 +63306539373730623464323137303134666534393536303330336335656235666630316666363039 +65663439376161313864386262643262326431666636643535613765343161353237353364636333 +63333664376335643739613435623034616230643561386533366538393836373562656138393563 +30366530313936363539646363363539646231386139626237333963333634393034663438386263 +35333534623335303034373236383564306135356334356138623061386638663261343830346262 +65623438333136623435323137656235323939613665373831656361383332323364633731316134 +35613064633332383238393430396465623962393932393262306132353764643135633437646233 +35396635303236343837333337666131363563613462326433363531663233646465633532386562 +36656165363137313635353534353530313139343165393239373734353564333332636135616166 +30623537303837653961306533656234326166333064633435633262356264303239386265313131 +30656237323639356265336362613930313436383362346661316634333765316662323038653235 +36633730363333653635663431623932303636396435643665643639376362646537623761393733 +33383161643733383662313461306530343164343439376162623963356337643237353465643830 +63356134666433356336333732366437613365623535333232346464653066323034313430373932 +61306639653065326531623436633832663264383562653030376435336336376365303139353831 +34663539306238396238373538613664616164653261353038656239363337666231643137663032 +61656132616666633161353138333963313936303530353831373766373864316130376433386431 +37646361653130616262666635323930363234323265383535646334383464323732366662316238 +64303662363935366133653364373439633935316132383736623330386432313132383461323438 +39386239646330326636633636316235333139343963633535333462393037373664346631393562 +35306235343164623237303930616363323438323766383835646261326539313239353062626336 +34393064353866346239363666306464666161316230646563326537343263303961326264663633 +35336435323038396435306465653636646336353762646266346465646135316633653831373239 +31376130663735633163376464373066633536653965376262313439336561353038346530343232 +38313661646631636632353363636661336336313366383863616566316435323139396530373438 +64623563653962333564643633313862383863666562373030376137393338323961383438663638 +30323534396432333533653635633935646663383162613133326363663834306138343038336366 +36373937623764616630323565663939663236376638343339376261336266663637323736363439 +37643238616138336361366562343031333466666165656435313037343766313030323963343739 +34376164613362303134366436356166666363663933653233373032613534613132383065646463 +38333630313237663235383635656430636362313639383131666135383938646335653965383366 +35363034633362366232626562323235353063643061353737363862343932363464363831626437 +31313635663731643331663136643837323031623266663963633734653439623331333230343364 +30643438343062353034623831393134343463643335616436353966346661663963343363663536 +32336338353330343862633561363463633961383865613162386261336135316335363634616166 +33663532666330623463646465336336623235383036313432623864353364373331323736643832 +35653234656263626565653461373461343161636539626231646462643532323066363930633333 +65626565636263323262393361396465613738383537323637373564633030386461383330653562 +30616262326334303062363739636339633033323530303762643735663366303739613431316463 +61323231663537303239653461623766326261656333366264313531643332633236666166626137 +33643862303737393030626331623335633337306439323961343936393361343134353230313735 +62313939663862326339653966386366343961323264643062396637373433306138346335356566 +30656563646138646335353364343036386630373035343536356539373664643539303163626638 +62303732646166323835303761313338326566623532613361363235366464306636353364336466 +65323362356330323735383862333433633735323264343034666264346537353762616337613639 +61643862616333656664343338653832386131633738363132623034623063663630636461363235 +62656539633933633839636565623866316430316236613134636135303131626630616366373136 +61656239363539336366643031393761643364366439336365653934333939313731313663643063 +33356564376565643034663665626161353236343764343737306233373534653435653036323935 +35616433313632306361626662363838386161303363663138303833613337373566313864633533 +61343032356531363439633166333531316531353239636166336566333364336235616663323864 +30663331626634653566613032643061656632323039633230376565346136643439303936373435 +33353638353165323865393337616231383330336662326133653630643638643761633866313335 +63313530666262613930353731393162656663383461616434643536666230626162316536643236 +62393166336133336335633963666437393633393537663065616331326261303436366336333061 +35646130366561366363316163333162323333333064613662323330643639313930376234333433 +39363462323165363163356137653664336365306538393232653436316664303235626234363831 +33323539613732383362326139373635386161393263393764656531313135626136343439313632 +36343035626433646361613637376166306536666230666435356332353661663661303964316563 +64656461313061346261383464373661396337663837353438646161626562636134656433333034 +39303463613732613833623865396362376437313431323435386633633533326363343139363537 +35386566666233386632646361633639333831643666663734626566653532383239616335626630 +31666566623434316462363832393634346561376265383439346363396165653233623666626466 +30383239373963303139363336383963343933323763623131393035323466626364636539373662 +32383431636533643131326530633132643531633766613834633131623935343737316563323238 +61646336656136613166356331353036613130616430353463323036373863373337393263653231 +35333765393731303766356133376635383364313438383836663939663063313038393866613365 +61653539653432383364356631326165356636326564623064363438333465306566323765373335 +38626437303331303466373539363135303337643631613736636665323038666434633764316539 +64396139336561336638303364363535323864653134646337333164363638376637306633323832 +36326231393165613530343965323337316664376263383733323432306165656338653438323034 +61363235333733373830376639646534646636333534613731313537363764376563353934396132 +38613734393433393233316539623839396331323735316561303038323733393065663165303464 +33383537316365656361346631613462643032656363303936366333306334663461613430376631 +62336663396165306166356236363063653233306139313730326662366435653136616462633434 +37323038396533326430373332666664313133633532326538626463663833373465656464363635 +65353731623962316465666435653232383834366638643135656330636339643062363135383339 +36323965396136373138386464393363373239383666313631633662353865636263656631343936 +36636261633436373031393061636532623064633634653239366161366336613037663764643537 +62343638373235343534383237353133333765616666323833363765653863636234353135386235 +30373530333432353635356664363563656638653031663632656532653431643831623932313962 +32323036303038363463343533623763373135373639306561623561663830393565613339373865 +33303431663634386335333831353238343138616463383963346336396433396136633231363830 +37386161613163353639636633323164303465313062303133623666646265343934656233636133 +31386239313433663461653935623564616162373031336633643039646132316135343433626166 +64376530393035653434326636393065333664636463343561303761666564626139626632303331 +36633062656365633736393230656339623365646638343532323764323563653739653434303036 +32386263376136663365353237386632653730396234636238336463616136643335303438313039 +37336562306637366163393231663639343561616266663338303439303035393766386235353063 +30383664346261356136626266656563306432626339356562393266623265376433383261323663 +65616163376566353734376633376136376431366535303538616566353463386365346163386231 +31666135616332346265303631656363343961393831646537636663386538313835383730383633 +62373038613763613034653438343164306366633766333663343663343134393030373763313739 +34376335383235353333666566303337333261373335396139666461633131373934316264333532 +32626536336663343532373662666664393034373063653164323936336532393263303565653133 +33663634303962646365656133333936353063383135616532343964336330653735393161323432 +38336162356632306434663537383662316437303361626361396365393732393263343035643433 +39373464626262323430306534326363346131626166353464363834383566346531653535653030 +32363133303637303830326235373030633931653062663335386665303833633839303266663931 +37333037633136663239323038393030366665326436623834393937316631663961643638616662 +39643761313265386430633765386238656431383737376161356630323130356535313930336434 +37343037363731303637386665343262643535363634613539383733653161313561663962386332 +37396338633236303030386336383433663166396363616530373834646133386564333835656136 +30313862663433393861656332643030393966303733626435373637303436613532343662626563 +62396435396562663836303465316134373232663562316232646237376238666465363261373430 +62646239393365646137343464643563666131346638363731393538313561313661316466356335 +37393838323839336264613031323833333661303763653464613133383538653831656262626533 +33356165653263316532363039336464316162313563373264393961613639613965636435663463 +32666537663533326366346166663164336538646665303563386466636337633961626164353937 +38333738323861656364393330663362626239326334346333613837333236633536366636663431 +33636633393037326664356630366264343930333566356664386465363264333931636464646236 +39323736343831303466386337623261323934383433623166616234643739643566343930373339 +65356237306662646264363266343261646463316635393136316338633761636365333134353133 +64333330313831353766346530656364643565353232626266353361386239396662363263383539 +36356431363062313962333538303235393933616465613161396262396566343065393835303330 +36643462303236623361356637656564396431613932343235373630353638643466623632383130 +66613562626333386530633365336239346431386335373934663239393032393265303134616462 +34323936396430303034373263366334396335663664646632303830356134643935643266333462 +30653734653238363866316434373130623432363732653238333732373733616666666464393636 +34353361663538343332353063356537663231646634333934383363613661623532376533633232 +36653734316630646339393763626632646139333935356434626230653635613834373264353632 +39353336383938303462643561336664376563326235383962376332396132346164626466346163 +62333838363039343531343237376636393566303330336431313230363464623634646230623632 +35373234376537383063383762333739353430316137353830323363313238336533636661633636 +63653666383838646130626432363235393334393861666533613336336636643432313032303233 +31646636313463303962336135373166623534333632623135376264653164343266653863313235 +61663432623264333631656537643136323833393838623261356163393630343738643565336636 +31383138656630626439313037313530323265306437386164663835383131373630653639323033 +61653030316566313236396131386532626135366263333231386334663132656434626137616334 +65626335633761653565303331623634646430356335323338323936323235313464396231333864 +31333333363233303463306333373037666562383932343665366661626464386634663534303330 +38396231396431316230326132383835306536383563393937333234633238333461303537386164 +66663136663031636461643438376131356165333065336632323338373064363035353065356134 +36326535346635343336363138353266366463393536366264613631623131343963393032666630 +66613166313435326261616365636535313565383432393965396266393065316634386431646132 +63303063646561363832656230633361633164626262663666386465366634633738323938613734 +36623234623263393130626635613163323762353166336439633663623731373564626536653363 +33633337303664343337383339323635356331663634343438363262343038373333343865303931 +33666561643365346539613463303037346638396464663362323962616538393133326364613963 +62626437333135346166333564326438333263396666306433313463613830313732393963323339 +36636130326332653637303732643463353934326264353738353039363631643831343163663261 +37643162343161343431333236643932323961613837663563363733616466316662363935306664 +65343431353233373435633565366264653233383434313436323336376630383334373233353433 +39613561353634653331346161643034373639656463356232303739663163363462326436373030 +38336661386262653333343365376262323733656233616164353731613333363662643439373061 +36393539353135333564303363623337396233636638386637636639646432376631353139316230 +39636462663164386439313961353032363833373566393565323539636561373363303562383464 +37653439326531356338343465626535346464356437376234393761666363616132373230633737 +30643662316233613530306466396436373933366361343536323136613032343964346266623937 +34326361343738383434373136333961363832643935393632663364373330373636383761326437 +65343832366361353630306164616637393733616231653832316366646630333064633063623436 +62373163323130346332333166343331343265653331643565646136316636633935646131313263 +31303365636130323133623232383136613235663066323261616561623935643938326433613635 +35636461376230613834353533653664653539373761366664663163663133303161393065313264 +63363461643330353362663165396331393061633766343661633262393230383461323737613138 +36663762393562333633613533333134363339653564393735333865666530643761353232633835 +31656632343831326361336437303965303538663832646537363565643033386337373537626634 +35616361643934326163306162353365623238663461356330333235633934313735306166636335 +38613436363531343334613030343737346632633966346633396334303432373934343061643533 +37633231333234333763366635323830383862346534333966326632666537326331353165396132 +32633730633031376662613833656538333433386237333831653032386464303333313034626361 +62373865636331383038353061663266336662303264633137616330333233646535356239363966 +30306163393334386632303530366265646365366335663639346464323463633964363633366466 +36353662623033383165313535363562643030376665333330373932663162396330386461636132 +63353136313563343239393866633265663562396630306263326532313636633538313032613936 +65353662373866353533323963323634306333373236343934376531376264363639393565366635 +36666133646665366632313164623330643337383638653734636564353862326236613631383632 +35316630383065626332643966656461303633633336643338393165383434353330313334346138 +34383430363337333439366664363339396662623837376562626461633838313661653935636539 +34393639363836306534616361376439663731353062363131326439336665303639316461626130 +35626165373963373662396234623662656562373262323861343238326263326432646434386261 +34636234333161333762643265396266653162613830353133653762386264326466613038386264 +62636439653939366336666362666162643131306562376635346134333635383462663966663636 +37653236653134313536646436613530353661633465633838663631333964323561346235613864 +37646636346232636234663364363336313630646162666162653439333936323365353933616564 +35313164393532623133626366643366363962303636316232383239323566643961393563643466 +31343064333839633838346266653162386239336234613865396162346638393435366136333739 +62353236346665343139343830363636393531333134363566336535333765646633376565323365 +31396264373136376565373632326437393566626233396565613230316461313632643135386564 +39316630663639386466383161646538643931646365653835613930383935316364363361636130 +35303965633436336337393134333239346663303564666435333864393431663338323633613065 +37386433636239326430386335313738373337663034343761623464646662633035313330653935 +39653764393564383161613966353835393063303631393035666264346661393663623438343264 +61343335656362346665306665343764666536643461376538383138336261646531626161323662 +64626139636432396564323338336263313062636163383732633666643264353662613934323137 +39646661643462376137376462366238313364313064613436646632386461303734343365663239 +64326364333439396234356236323663653966303135383134613531366565616634326132653061 +39323532306339393666333036313132616561363261623162386436643765393961323238633361 +66356265353337303865343563383464313734396166303062323462343766356565386134386334 +31613563363962663533303133623733323436383631316433636136353437353835393735396435 +32623433623561326166633763336136323166333430393331306530393631636663656466363431 +6130313132383162626264393061623966643633376535366662 |