CSIT Azure integration

+ Terraform
+ Ansible

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: I353511c5ef39d8c8d43fedc61c8dbb30b4821e1b

4 files changed, 83 insertions, 317 deletions

diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/kernel_install.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/kernel_install.yaml
deleted file mode 100644
index 4f7800b249..0000000000
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/kernel_install.yaml
+++ /dev/null
@@ -1,81 +0,0 @@
----
-# file: roles/common/tasks/kernel_install.yaml
-
-- name: Kernel VM - Backup remote initramfs modules
-  copy:
-    src: '/etc/initramfs-tools/modules'
-    dest: '/tmp/initramfs_modules.bkp'
-    remote_src: yes
-  ignore_errors: yes
-  register: __initramfs_modules_backuped
-  tags: install-kernel-image
-
-- name: Kernel VM - Backup remote initramfs resume config
-  copy:
-    src: '/etc/initramfs-tools/conf.d/resume'
-    dest: '/tmp/initramfs-resume.bkp'
-    remote_src: yes
-  ignore_errors: yes
-  register: __initramfs_resume_backuped
-  tags: install-kernel-image
-
-- name: Kernel VM - Update remote initramfs modules
-  copy:
-    src: '../files/initramfs_modules'
-    dest: '/etc/initramfs-tools/modules'
-  tags: install-kernel-image
-
-- name: Kernel VM - Update remote initramfs resume config
-  copy:
-    src: '../files/initramfs_resume'
-    dest: '/etc/initramfs-tools/conf.d/resume'
-  tags: install-kernel-image
-
-- name: Kernel VM - Create target kernel dir
-  file:
-    path: '/opt/boot'
-    state: 'directory'
-  tags: install-kernel-image
-
-- name: Kernel VM - Build initrd image
-  shell: 'update-initramfs -k {{ ansible_kernel }} -c -b /opt/boot'
-  tags: install-kernel-image
-
-- name: Kernel VM - Copy corresponding kernel img
-  copy:
-    src: '/boot/vmlinuz-{{ ansible_kernel }}'
-    dest: '/opt/boot/vmlinuz-{{ ansible_kernel }}'
-    remote_src: yes
-  tags: install-kernel-image
-
-- name: Kernel VM - Restore remote initramfs modules
-  copy:
-    src: '/tmp/initramfs_modules.bkp'
-    dest: '/etc/initramfs-tools/modules'
-    remote_src: yes
-  ignore_errors: yes
-  when: __initramfs_modules_backuped
-  tags: install-kernel-image
-
-- name: Kernel VM - Remove remote backup initramfs modules
-  file:
-    path: '/tmp/initramfs_modules.bkp'
-    state: 'absent'
-  when: __initramfs_modules_backuped
-  tags: install-kernel-image
-
-- name: Kernel VM - Restore remote initramfs resume config
-  copy:
-    src: '/tmp/initramfs-resume.bkp'
-    dest: '/etc/initramfs-tools/conf.d/resume'
-    remote_src: yes
-  ignore_errors: yes
-  when: __initramfs_resume_backuped
-  tags: install-kernel-image
-
-- name: Kernel VM - Remove remote backup initramfs resume config
-  file:
-    path: '/tmp/initramfs-resume.bkp'
-    state: 'absent'
-  when: __initramfs_resume_backuped
-  tags: install-kernel-image
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index 84358bf863..e60eec9fb0 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -1,166 +1,106 @@
 ---
 # file: roles/common/tasks/main.yaml
 
-- name: Ensure the system exists in Cobbler
-  cobbler_system:
-    host: '{{ cobbler_hostname }}'
-    port: 60080
-    interfaces:
-      br1:
-        ipaddress: '{{ ansible_default_ipv4.address }}'
-        macaddress: '{{ ansible_default_ipv4.macaddress }}'
-    name: '{{ hostname }}'
-    password: '{{ cobbler_password }}'
-    properties:
-      hostname: '{{ hostname }}'
-      gateway: '{{ ansible_default_ipv4.gateway }}'
-      profile: '{{ cobbler_profile }}'
-      name_servers: '{{ name_servers }}'
-      name_servers_search: '{{ name_servers_search }}'
-      kickstart: '/var/lib/cobbler/kickstarts/{{ cobbler_profile }}.seed'
-      kernel_options: '"interface={{ ansible_default_ipv4.interface }}"'
-      netboot_enabled: yes
-    username: '{{ cobbler_username }}'
-    use_ssl: no
-    validate_certs: no
-  when: provision_enabled
-  delegate_to: localhost
-  tags: cobbler-include
-
-- name: Commit Cobbler changes
-  cobbler_sync:
-    host: '{{ cobbler_hostname }}'
-    port: 60080
-    password: '{{ cobbler_password }}'
-    username: '{{ cobbler_username }}'
-    use_ssl: no
-    validate_certs: no
-  run_once: yes
-  when: provision_enabled
-  delegate_to: localhost
-  register: __included_in_cobbler
-  notify:
-    - 'Boot from network'
-    - 'Reboot server'
-  tags: cobbler-include
-
-- meta: flush_handlers
-
-- name: Copy netplan network config file
-  template:
-    src: 'files/netplan_config'
-    dest: '/etc/netplan/01-netcfg.yaml'
-    owner: 'root'
-    group: 'root'
-    mode: '0644'
-  tags: copy-interface-file
-
 - name: Add permanent proxy settings
   lineinfile:
-    path: '/etc/environment'
-    state: 'present'
-    line: '{{ item.key }}={{ item.value }}'
-  with_dict: '{{ proxy_env }}'
+    path: "/etc/environment"
+    state: "present"
+    line: "{{ item.key }}={{ item.value }}"
+  with_dict: "{{ proxy_env }}"
   when: proxy_env is defined
+  tags:
+    - set-proxy
 
-- name: Install distribution - release - machine prerequisites
-  include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml'
-  tags: [ install-csit-dependencies, copy-apt-sources ]
+- name: Install Distribution - Release - Machine Prerequisites
+  package:
+    name: "{{ packages | flatten(levels=1) }}"
+    state: latest
+    update_cache: true
+  tags:
+    - install-dependencies
 
 - name: Install CSIT PIP requirements
   pip:
     name:
-      - 'ecdsa==0.13.3'
-      - 'paramiko==2.6.0'
-      - 'pycrypto==2.6.1'
-      - 'pypcap==1.2.3'
-      - 'PyYAML==5.1.1'
-      - 'requests==2.22.0'
-      - 'robotframework==3.1.2'
-      - 'scapy==2.4.3'
-      - 'scp==0.13.2'
-      - 'ansible==2.7.8'
-      - 'dill==0.2.8.2'
-      - 'numpy==1.17.3'
-      - 'hdrhistogram==0.6.1'
-      - 'pandas==0.25.3'
-      - 'plotly==4.1.1'
-      - 'PTable==0.9.2'
-      - 'Sphinx==2.2.1'
-      - 'sphinx-rtd-theme==0.4.0'
-      - 'sphinxcontrib-programoutput==0.15'
-      - 'sphinxcontrib-robotdoc==0.11.0'
-      - 'alabaster==0.7.12'
-      - 'Babel==2.7.0'
-      - 'bcrypt==3.1.7'
-      - 'certifi==2019.9.11'
-      - 'cffi==1.13.2'
-      - 'chardet==3.0.4'
-      - 'cryptography==2.8'
-      - 'docutils==0.15.2'
-      - 'future==0.18.2'
-      - 'idna==2.8'
-      - 'imagesize==1.1.0'
-      - 'Jinja2==2.10.3'
-      - 'MarkupSafe==1.1.1'
-      - 'packaging==19.2'
-      - 'pbr==5.4.3'
-      - 'pycparser==2.19'
-      - 'Pygments==2.4.2'
-      - 'PyNaCl==1.3.0'
-      - 'pyparsing==2.4.4'
-      - 'python-dateutil==2.8.1'
-      - 'pytz==2019.3'
-      - 'retrying==1.3.3'
-      - 'six==1.13.0'
-      - 'snowballstemmer==2.0.0'
-      - 'sphinxcontrib-applehelp==1.0.1'
-      - 'sphinxcontrib-devhelp==1.0.1'
-      - 'sphinxcontrib-htmlhelp==1.0.2'
-      - 'sphinxcontrib-jsmath==1.0.1'
-      - 'sphinxcontrib-qthelp==1.0.2'
-      - 'sphinxcontrib-serializinghtml==1.1.3'
-      - 'urllib3==1.25.6'
-  tags: install-pip
+      - "ecdsa==0.13.3"
+      - "paramiko==2.6.0"
+      - "pycrypto==2.6.1"
+      - "pypcap==1.2.3"
+      - "PyYAML==5.1.1"
+      - "requests==2.22.0"
+      - "robotframework==3.1.2"
+      - "scapy==2.4.3"
+      - "scp==0.13.2"
+      - "ansible==2.7.8"
+      - "dill==0.2.8.2"
+      - "numpy==1.17.3"
+      - "hdrhistogram==0.6.1"
+      - "pandas==0.25.3"
+      - "plotly==4.1.1"
+      - "PTable==0.9.2"
+      - "Sphinx==2.2.1"
+      - "sphinx-rtd-theme==0.4.0"
+      - "sphinxcontrib-programoutput==0.15"
+      - "sphinxcontrib-robotdoc==0.11.0"
+      - "alabaster==0.7.12"
+      - "Babel==2.7.0"
+      - "bcrypt==3.1.7"
+      - "certifi==2019.9.11"
+      - "cffi==1.13.2"
+      - "chardet==3.0.4"
+      - "cryptography==2.8"
+      - "docutils==0.15.2"
+      - "future==0.18.2"
+      - "idna==2.8"
+      - "imagesize==1.1.0"
+      - "Jinja2==2.10.3"
+      - "MarkupSafe==1.1.1"
+      - "packaging==19.2"
+      - "pbr==5.4.3"
+      - "pycparser==2.19"
+      - "Pygments==2.4.2"
+      - "PyNaCl==1.3.0"
+      - "pyparsing==2.4.4"
+      - "python-dateutil==2.8.1"
+      - "pytz==2019.3"
+      - "retrying==1.3.3"
+      - "six==1.13.0"
+      - "snowballstemmer==2.0.0"
+      - "sphinxcontrib-applehelp==1.0.1"
+      - "sphinxcontrib-devhelp==1.0.1"
+      - "sphinxcontrib-htmlhelp==1.0.2"
+      - "sphinxcontrib-jsmath==1.0.1"
+      - "sphinxcontrib-qthelp==1.0.2"
+      - "sphinxcontrib-serializinghtml==1.1.3"
+      - "urllib3==1.25.6"
+  tags:
+    - install-pip
 
 - name: Install CSIT PIP requirements - SciPy workaround
   pip:
     name:
-      - 'scipy==1.1.0'
-  tags: install-pip
+      - "scipy==1.1.0"
+  tags:
+    - install-pip
 
 - name: Set sudoers admin
   lineinfile:
-    path: '/etc/sudoers'
-    state: 'present'
-    regexp: '^%admin ALL='
-    line: '%admin ALL=(ALL) ALL'
-    validate: '/usr/sbin/visudo -cf %s'
-  tags: set-sudoers
+    path: "/etc/sudoers"
+    state: "present"
+    regexp: "^%admin ALL="
+    line: "%admin ALL=(ALL) ALL"
+    validate: "/usr/sbin/visudo -cf %s"
+  tags:
+    - set-sudoers
 
 - name: Set sudoers sudo
   lineinfile:
-    path: '/etc/sudoers'
-    state: 'present'
-    regexp: '^%sudo'
-    line: '%sudo ALL=(ALL:ALL) NOPASSWD: ALL'
-    validate: '/usr/sbin/visudo -cf %s'
-  tags: set-sudoers
-
-- name: Copy grub file
-  template:
-   src: 'files/grub_{{ ansible_machine }}'
-   dest: '/etc/default/grub'
-   owner: 'root'
-   group: 'root'
-   mode: '644'
-  notify:
-    - 'Update GRUB'
-    - 'Reboot server'
-  tags: copy-grub
-
-- name: Mellanox Install - Driver
-  import_tasks: mellanox.yaml
-  tags: mellanox-install
+    path: "/etc/sudoers"
+    state: "present"
+    regexp: "^%sudo"
+    line: "%sudo ALL=(ALL:ALL) NOPASSWD: ALL"
+    validate: "/usr/sbin/visudo -cf %s"
+  tags:
+    - set-sudoers
 
 - meta: flush_handlers
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/mellanox.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/mellanox.yaml
deleted file mode 100644
index b8d14ad67c..0000000000
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/mellanox.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
----
-# file: roles/common/tasks/mellanox.yaml
-
-- name: Mellanox Install - Check Presence of Mellanox Hardware
-  shell: "lspci | grep Mellanox | awk '{print $1}'"
-  register: mellanox_pcis
-  failed_when: no
-  changed_when: no
-  tags: install-mellanox
-
-- name: Mellanox Install - Get OFED
-  get_url:
-    url: 'http://content.mellanox.com/ofed/MLNX_OFED-{{ mellanox.version }}/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz'
-    dest: '/opt/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz'
-    mode: 0644
-  when: mellanox_pcis.stdout_lines | length > 0
-  tags: install-mellanox
-
-- name: Mellanox Install - Extract OFED
-  unarchive:
-    remote_src: true
-    src: '/opt/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz'
-    dest: '/opt/'
-    creates: '/opt/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}'
-  register: mellanox_firmware_extracted
-  when: mellanox_pcis.stdout_lines | length > 0
-  tags: install-mellanox
-
-- name: Mellanox Install - Install OFED
-  command: './mlnxofedinstall --with-mft --dpdk --force --upstream-libs'
-  args:
-    chdir: '/opt/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}'
-  when: mellanox_pcis.stdout_lines | length > 0 and mellanox_firmware_extracted
-  tags: install-mellanox
-
-- name: Mellanox Install - Switch Infiniband to Ethernet
-  command: 'mlxconfig --yes --dev {{ item }} set LINK_TYPE_P1=2 LINK_TYPE_P2=2'
-  with_items: "{{ mellanox_pcis.stdout_lines }}"
-  tags: install-mellanox
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_bionic.yaml
deleted file mode 100644
index 480358b0a3..0000000000
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_bionic.yaml
+++ /dev/null
@@ -1,54 +0,0 @@
----
-# file: roles/common/tasks/ubuntu_bionic.yaml
-
-- name: Copy apt sources file
-  template:
-    src: 'files/apt-sources_{{ ansible_distribution_release }}_{{ ansible_machine }}.list'
-    dest: '/etc/apt/sources.list'
-  tags: copy-apt-sources
-
-- name: Install CSIT dependencies
-  apt:
-    name:
-      - 'autoconf'
-      - 'build-essential'
-      - 'cgroup-bin'
-      - 'dkms'
-      - 'iperf3'
-      - 'libpcap-dev'
-      - 'python-all'
-      - 'python-apt'
-      - 'python-cffi'
-      - 'python-cffi-backend'
-      - 'python-dev'
-      - 'python-pip'
-      - 'python-setuptools'
-      - 'python3-all'
-      - 'python3-apt'
-      - 'python3-cffi'
-      - 'python3-cffi-backend'
-      - 'python3-dev'
-      - 'python3-pip'
-      - 'python3-setuptools'
-      - 'qemu-system'
-      - 'socat'
-      - 'virtualenv'
-      - 'unzip'
-      - 'zlib1g-dev'
-    state: 'present'
-    cache_valid_time: 3600
-    install_recommends: False
-  tags: install-csit-dependencies
-
-- name: Install CSIT dependencies - aarch64 SciPy
-  apt:
-    name:
-      - 'gfortran'
-      - 'libblas-dev'
-      - 'liblapack-dev'
-    state: 'present'
-    cache_valid_time: 3600
-    install_recommends: False
-  when: >
-    ansible_machine == 'aarch64'
-  tags: install-csit-dependencies