Ansible: Mellanox install

+ OFED
+ ibdev2netdev
- aligning few naming conventions

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: I0dff28b2cd4bbc4753597366d9b03b497fa336c4

8 files changed, 152 insertions, 105 deletions

diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
index 062f3af10e..68994ec36b 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
@@ -8,7 +8,7 @@ provision_enabled: False
 # name_servers_search is used in /etc/hosts file on target machine.
 name_servers_search: 'linuxfoundation.org'
 # name_servers is used in /etc/netplan/01-netcfg.yaml
-name_servers: "199.204.44.24, 199.204.47.54"
+name_servers: "1.1.1.1, 8.8.8.8"
 
 # Proxy settings: Uncomment and fill the proper values. These variables will be
 # set globally by writing into /etc/environment file on target machine.
@@ -45,7 +45,6 @@ kubernetes:
 trex:
   target_dir: '/opt'
   version: '2.61'
-  url: 'https://github.com/cisco-system-traffic-generator/trex-core/archive'
 
 # DPDK settings.
 dpdk:
@@ -60,10 +59,13 @@ dpdk:
 wrk:
   target_dir: '/opt'
   version: '4.0.2'
-  url: 'https://github.com/wg/wrk/archive'
 
 # Calibration settings.
 jitter:
   directory: '/tmp/pma_tools'
   core: 7
   iterations: 30
+
+# Mellanox OFED settings.
+mellanox:
+  version: '4.6-1.0.1.1'
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index 915fc0bbd0..6bb4d4614e 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -97,4 +97,8 @@
     - 'Reboot server'
   tags: copy-grub
 
+- name: Mellanox Install - Driver
+  import_tasks: mellanox.yaml
+  tags: mellanox-install
+
 - meta: flush_handlers
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/mellanox.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/mellanox.yaml
new file mode 100644
index 0000000000..b8d14ad67c
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/mellanox.yaml
@@ -0,0 +1,39 @@
+---
+# file: roles/common/tasks/mellanox.yaml
+
+- name: Mellanox Install - Check Presence of Mellanox Hardware
+  shell: "lspci | grep Mellanox | awk '{print $1}'"
+  register: mellanox_pcis
+  failed_when: no
+  changed_when: no
+  tags: install-mellanox
+
+- name: Mellanox Install - Get OFED
+  get_url:
+    url: 'http://content.mellanox.com/ofed/MLNX_OFED-{{ mellanox.version }}/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz'
+    dest: '/opt/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz'
+    mode: 0644
+  when: mellanox_pcis.stdout_lines | length > 0
+  tags: install-mellanox
+
+- name: Mellanox Install - Extract OFED
+  unarchive:
+    remote_src: true
+    src: '/opt/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz'
+    dest: '/opt/'
+    creates: '/opt/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}'
+  register: mellanox_firmware_extracted
+  when: mellanox_pcis.stdout_lines | length > 0
+  tags: install-mellanox
+
+- name: Mellanox Install - Install OFED
+  command: './mlnxofedinstall --with-mft --dpdk --force --upstream-libs'
+  args:
+    chdir: '/opt/MLNX_OFED_LINUX-{{ mellanox.version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}'
+  when: mellanox_pcis.stdout_lines | length > 0 and mellanox_firmware_extracted
+  tags: install-mellanox
+
+- name: Mellanox Install - Switch Infiniband to Ethernet
+  command: 'mlxconfig --yes --dev {{ item }} set LINK_TYPE_P1=2 LINK_TYPE_P2=2'
+  with_items: "{{ mellanox_pcis.stdout_lines }}"
+  tags: install-mellanox
diff --git a/resources/tools/testbed-setup/ansible/roles/tg/tasks/trex.yaml b/resources/tools/testbed-setup/ansible/roles/tg/tasks/trex.yaml
index 3cfcbea825..03f7d6da53 100644
--- a/resources/tools/testbed-setup/ansible/roles/tg/tasks/trex.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/tg/tasks/trex.yaml
@@ -1,40 +1,41 @@
 ---
 # file: roles/tg/tasks/trex.yaml
 
-- name: Download TRex release archive
+- name: T-Rex Install - Get Release Archive
   get_url:
-    url: '{{ trex.url }}/v{{ trex.version }}.tar.gz'
+    url: 'https://github.com/cisco-system-traffic-generator/trex-core/archive/v{{ trex.version }}.tar.gz'
     dest: '{{ trex.target_dir }}/trex-core-{{ trex.version }}.tar.gz'
     mode: 0644
-  register: 'linux__trex_downloaded'
+  register: trex_downloaded
   tags: install-trex
 
-- name: Ensure TRex directory exists
+- name: T-Rex Install - Ensure Directory Exists
   file:
     path: '{{ trex.target_dir }}/trex-core-{{ trex.version }}'
     state: 'directory'
-  register: 'linux__trex_dir_created'
+  register: trex_dir_created
   tags: install-trex
 
-- name: Extract TRex release archive
-  become: yes
+- name: T-Rex Install - Extract Release Archive
   unarchive:
+    remote_src: true
     src: '{{ trex.target_dir }}/trex-core-{{ trex.version }}.tar.gz'
     dest: '{{ trex.target_dir }}/'
-    creates: '{{ trex.target_dir }}/trex-core-{{ trex.version }}/linux_dpdk'
-    remote_src: yes
-  when: 'linux__trex_dir_created'
-  register: 'linux__trex_extracted'
+    creates: '{{ trex.target_dir }}/trex-core-{{ trex.version }}/linux_dpdk/'
+  when: trex_dir_created
+  register: trex_extracted
   tags: install-trex
 
-- name: Compile TRex release I
-  become: yes
-  shell: 'cd {{ trex.target_dir }}/trex-core-{{ trex.version }}/linux_dpdk/; ./b configure; ./b build'
-  when: 'linux__trex_extracted'
+- name: T-Rex Install - Compile Release I
+  command: './b configure; ./b build'
+  args:
+    chdir: '{{ trex.target_dir }}/trex-core-{{ trex.version }}/linux_dpdk/'
+  when: trex_extracted
   tags: install-trex
 
-- name: Compile TRex release II
-  become: yes
-  shell: 'cd {{ trex.target_dir }}/trex-core-{{ trex.version }}/scripts/ko/src; make; make install'
-  when: 'linux__trex_extracted'
+- name: T-Rex Install - Compile Release II
+  command: 'make; make install'
+  args:
+    chdir: '{{ trex.target_dir }}/trex-core-{{ trex.version }}/scripts/ko/src'
+  when: trex_extracted
   tags: install-trex
diff --git a/resources/tools/testbed-setup/ansible/roles/tg/tasks/wrk.yaml b/resources/tools/testbed-setup/ansible/roles/tg/tasks/wrk.yaml
index e7d22e1aa1..aee3761476 100644
--- a/resources/tools/testbed-setup/ansible/roles/tg/tasks/wrk.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/tg/tasks/wrk.yaml
@@ -1,41 +1,40 @@
 ---
 # file: roles/tg/tasks/wrk.yaml
 
-- name: Download WRK release archive
+- name: WRK Install - Get Release Archive
   get_url:
-    url: '{{ wrk.url }}/{{ wrk.version }}.tar.gz'
+    url: 'https://github.com/wg/wrk/archive/{{ wrk.version }}.tar.gz'
     dest: '{{ wrk.target_dir }}/{{ wrk.version }}.tar.gz'
     mode: 0644
-  register: 'linux__wrk_downloaded'
+  register: wrk_downloaded
   tags: install-wrk
 
-- name: Ensure WRK directory exists
+- name: WRK Install - Ensure Directory Exists
   file:
     path: '{{ wrk.target_dir }}/wrk-{{ wrk.version }}'
     state: 'directory'
-  register: 'linux__wrk_dir_created'
+  register: wrk_dir_created
   tags: install-wrk
 
-- name: Extract WRK release archive
-  become: yes
+- name: WRK Install - Extract Release Archive
   unarchive:
+    remote_src: true
     src: '{{ wrk.target_dir }}/{{ wrk.version }}.tar.gz'
     dest: '{{ wrk.target_dir }}/'
     creates: '{{ wrk.target_dir }}/wrk-{{ wrk.version }}/src'
-    remote_src: yes
-  when: 'linux__wrk_dir_created'
-  register: 'linux__wrk_extracted'
+  when: wrk_dir_created
+  register: wrk_extracted
   tags: install-wrk
 
-- name: Compile WRK release
-  become: yes
-  shell: 'cd {{ wrk.target_dir }}/wrk-{{ wrk.version }}; make'
-  when: 'linux__wrk_extracted'
-  register: 'linux__wrk_compiled'
+- name: WRK Install - Compile Release
+  command: 'make'
+  args:
+    chdir: '{{ wrk.target_dir }}/wrk-{{ wrk.version }}'
+  when: wrk_extracted
+  register: wrk_compiled
   tags: install-wrk
 
-- name: Move WRK binary
-  become: yes
+- name: WRK Install - Move Binary
   command: 'mv {{ wrk.target_dir }}/wrk-{{ wrk.version }}/wrk /usr/local/bin/'
-  when: 'linux__wrk_compiled'
+  when: wrk_compiled
   tags: install-wrk
diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml
index 39a401a7de..bb8c67660a 100644
--- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml
@@ -11,7 +11,9 @@
 
 - name: Turbo boost specific
   import_tasks: turbo_boost.yaml
-  when: cpu_microarchitecture == "skylake" or cpu_microarchitecture == "cascadelake"
+  when: >
+    cpu_microarchitecture == "skylake" or
+    cpu_microarchitecture == "cascadelake"
   tags: [ install-msr, disable-turbo-boost ]
 
 - name: Copy CSIT sysctl file
diff --git a/topologies/available/lf_2n_clx_testbed27.yaml b/topologies/available/lf_2n_clx_testbed27.yaml
index d820353266..8f24a53c88 100644
--- a/topologies/available/lf_2n_clx_testbed27.yaml
+++ b/topologies/available/lf_2n_clx_testbed27.yaml
@@ -64,22 +64,22 @@ nodes:
         driver: i40e
         link: link6
         model: Intel-XXV710
-#      port7:
-#        # s34-t27-tg1-c9/p1 - 100GE-port1 ConnectX5-2p100GE.
-#        mac_address: "?"
-#        pci_address: "0000:5e:00.0"
-#        ip4_address: "172.16.70.2"
-#        driver: mlx5_core
-#        link: link7
-#        model: Mellanox-CX556A
-#      port8:
-#        # s34-t27-tg1-c9/p2 - 100GE-port2 ConnectX5-2p100GE.
-#        mac_address: "?"
-#        pci_address: "0000:5e:00.1"
-#        ip4_address: "172.16.70.1"
-#        driver: mlx5_core
-#        link: link8
-#        model: Mellanox-CX556A
+      port7:
+        # s34-t27-tg1-c9/p1 - 100GE-port1 ConnectX5-2p100GE.
+        mac_address: "ec:0d:9a:8c:c6:a6"
+        pci_address: "0000:5e:00.0"
+        ip4_address: "172.16.70.2"
+        driver: mlx5_core
+        link: link7
+        model: Mellanox-CX556A
+      port8:
+        # s34-t27-tg1-c9/p2 - 100GE-port2 ConnectX5-2p100GE.
+        mac_address: "ec:0d:9a:8c:c6:a7"
+        pci_address: "0000:5e:00.1"
+        ip4_address: "172.16.70.1"
+        driver: mlx5_core
+        link: link8
+        model: Mellanox-CX556A
   DUT1:
     type: DUT
     host: "10.32.8.18"
@@ -142,19 +142,19 @@ nodes:
         driver: i40e
         link: link6
         model: Intel-XXV710
-#      port7:
-#        # s33-t27-sut1-c9/p1 - 100GE-port1 ConnectX5-2p100GE.
-#        mac_address: "?"
-#        pci_address: "0000:5e:00.0"
-#        ip4_address: "172.16.70.2"
-#        driver: mlx5_core
-#        link: link7
-#        model: Mellanox-CX556A
-#      port8:
-#        # s33-t27-sut1-c9/p2 - 100GE-port2 ConnectX5-2p100GE.
-#        mac_address: "?"
-#        pci_address: "0000:5e:00.1"
-#        ip4_address: "172.16.70.1"
-#        driver: mlx5_core
-#        link: link8
-#        model: Mellanox-CX556A
+      port7:
+        # s33-t27-sut1-c9/p1 - 100GE-port1 ConnectX5-2p100GE.
+        mac_address: "ec:0d:9a:8c:c6:96"
+        pci_address: "0000:5e:00.0"
+        ip4_address: "172.16.70.2"
+        driver: mlx5_core
+        link: link7
+        model: Mellanox-CX556A
+      port8:
+        # s33-t27-sut1-c9/p2 - 100GE-port2 ConnectX5-2p100GE.
+        mac_address: "ec:0d:9a:8c:c6:96"
+        pci_address: "0000:5e:00.1"
+        ip4_address: "172.16.70.1"
+        driver: mlx5_core
+        link: link8
+        model: Mellanox-CX556A
diff --git a/topologies/available/lf_2n_clx_testbed28.yaml b/topologies/available/lf_2n_clx_testbed28.yaml
index 98a34cf7c9..07d13a362d 100644
--- a/topologies/available/lf_2n_clx_testbed28.yaml
+++ b/topologies/available/lf_2n_clx_testbed28.yaml
@@ -64,22 +64,22 @@ nodes:
         driver: i40e
         link: link6
         model: Intel-XXV710
-#      port7:
-#        # s36-t28-tg1-c9/p1 - 100GE-port1 ConnectX5-2p100GE.
-#        mac_address: "?"
-#        pci_address: "0000:5e:00.0"
-#        ip4_address: "172.16.70.2"
-#        driver: mlx5_core
-#        link: link7
-#        model: Mellanox-CX556A
-#      port8:
-#        # s36-t28-tg1-c9/p2 - 100GE-port2 ConnectX5-2p100GE.
-#        mac_address: "?"
-#        pci_address: "0000:5e:00.1"
-#        ip4_address: "172.16.70.1"
-#        driver: mlx5_core
-#        link: link8
-#        model: Mellanox-CX556A
+      port7:
+        # s36-t28-tg1-c9/p1 - 100GE-port1 ConnectX5-2p100GE.
+        mac_address: "ec:0d:9a:8c:c7:f6"
+        pci_address: "0000:5e:00.0"
+        ip4_address: "172.16.70.2"
+        driver: mlx5_core
+        link: link7
+        model: Mellanox-CX556A
+      port8:
+        # s36-t28-tg1-c9/p2 - 100GE-port2 ConnectX5-2p100GE.
+        mac_address: "ec:0d:9a:8c:c7:f7"
+        pci_address: "0000:5e:00.1"
+        ip4_address: "172.16.70.1"
+        driver: mlx5_core
+        link: link8
+        model: Mellanox-CX556A
   DUT1:
     type: DUT
     host: "10.32.8.20"
@@ -142,19 +142,19 @@ nodes:
         driver: i40e
         link: link6
         model: Intel-XXV710
-#      port7:
-#        # s35-t28-sut1-c9/p1 - 100GE-port1 ConnectX5-2p100GE.
-#        mac_address: "?"
-#        pci_address: "0000:5e:00.0"
-#        ip4_address: "172.16.70.2"
-#        driver: mlx5_core
-#        link: link7
-#        model: Mellanox-CX556A
-#      port8:
-#        # s35-t28-sut1-c9/p2 - 100GE-port2 ConnectX5-2p100GE.
-#        mac_address: "?"
-#        pci_address: "0000:5e:00.1"
-#        ip4_address: "172.16.70.1"
-#        driver: mlx5_core
-#        link: link8
-#        model: Mellanox-CX556A
+      port7:
+        # s35-t28-sut1-c9/p1 - 100GE-port1 ConnectX5-2p100GE.
+        mac_address: "ec:0d:9a:8c:c7:b6"
+        pci_address: "0000:5e:00.0"
+        ip4_address: "172.16.70.2"
+        driver: mlx5_core
+        link: link7
+        model: Mellanox-CX556A
+      port8:
+        # s35-t28-sut1-c9/p2 - 100GE-port2 ConnectX5-2p100GE.
+        mac_address: "ec:0d:9a:8c:c7:b7"
+        pci_address: "0000:5e:00.1"
+        ip4_address: "172.16.70.1"
+        driver: mlx5_core
+        link: link8
+        model: Mellanox-CX556A