Infra: Move probes under ansible instead of terraform

+ More stable probe handling.
+ Naming cleanup due to errors.

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: I3bb1237af20636919f869f2eee53597202d00792

21 files changed, 271 insertions, 69 deletions

diff --git a/resources/tools/testbed-setup/ansible/nomad.yaml b/resources/tools/testbed-setup/ansible/nomad.yaml
index 653215651a..134fffcb33 100644
--- a/resources/tools/testbed-setup/ansible/nomad.yaml
+++ b/resources/tools/testbed-setup/ansible/nomad.yaml
@@ -15,4 +15,8 @@
     - role: nomad
       tags: nomad
     - role: consul
-      tags: consul
\ No newline at end of file
+      tags: consul
+    - role: prometheus_exporter
+      tags: prometheus_exporter
+    - role: cadvisor
+      tags: cadvisor
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/ab/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/ab/tasks/main.yaml
index c422b4d2e0..85003d583d 100644
--- a/resources/tools/testbed-setup/ansible/roles/ab/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/ab/tasks/main.yaml
@@ -1,15 +1,18 @@
 ---
 # file: roles/ab/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - ab-inst-prerequisites
 
-- name: Install Apache ab tools
+- name: Inst - Apache ab tools
   package:
     name: "{{ packages | flatten(levels=1) }}"
     state: present
-  tags: install-ab
+  tags:
+    - ab-inst-ab
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/cadvisor/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/cadvisor/defaults/main.yaml
new file mode 100644
index 0000000000..3b25e551ea
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/cadvisor/defaults/main.yaml
@@ -0,0 +1,24 @@
+---
+# file: roles/cadvisor/defaults/main.yaml
+
+packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}"
+
+packages_base:
+  - []
+
+packages_by_distro:
+  ubuntu:
+    - "python3-docker"
+    - "python3-dockerpty"
+
+packages_by_arch:
+  aarch64:
+    - []
+  x86_64:
+    - []
+
+image: "{{ image_by_arch[ansible_machine] }}"
+
+image_by_arch:
+  aarch64: "zcube/cadvisor:v0.37.0"
+  x86_64: "gcr.io/cadvisor/cadvisor:v0.38.7"
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/cadvisor/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/cadvisor/tasks/main.yaml
new file mode 100644
index 0000000000..a2a13368c2
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/cadvisor/tasks/main.yaml
@@ -0,0 +1,39 @@
+---
+# file: roles/cadvisor/tasks/main.yaml
+
+- name: Inst - Update Package Cache (APT)
+  apt:
+    update_cache: yes
+    cache_valid_time: 3600
+  when:
+    - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - cadvisor-inst-prerequisites
+
+- name: Inst - Prerequisites
+  package:
+    name: "{{ packages | flatten(levels=1) }}"
+    state: latest
+  tags:
+    - cadvisor-inst-prerequisites
+
+- name: Inst - Start a container
+  docker_container:
+    name: "cAdvisor"
+    image: "{{ image }}"
+    state: "started"
+    restart_policy: "unless-stopped"
+    detach: yes
+    devices:
+     - "/dev/kmsg"
+    ports:
+     - "8080:8080"
+    privileged: yes
+    volumes:
+      - "/:/rootfs:ro"
+      - "/var/run:/var/run:ro"
+      - "/sys:/sys:ro"
+      - "/var/lib/docker/:/var/lib/docker:ro"
+      - "/dev/disk/:/dev/disk:ro"
+  tags:
+    - cadvisor-run-container
diff --git a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
index 877304cc36..5d0c3b18ff 100644
--- a/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/calibration/tasks/main.yaml
@@ -1,19 +1,21 @@
 ---
 # file: roles/calibration/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - calibration-inst-prerequisites
 
-- name: Install Distribution - Release - Machine Prerequisites
+- name: Inst - Prerequisites
   package:
     name: "{{ packages | flatten(levels=1) }}"
     state: latest
   tags:
-    - install-dependencies
+    - calibration-inst-prerequisites
 
 - name: Check CPU Power States
   shell: "lscpu"
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index 7be6f73951..35368dd45e 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -11,19 +11,21 @@
   tags:
     - set-proxy
 
-- Name: Update package cache (apt)
+- name: Inst - Update package cache (apt)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - common-inst-prerequisites
 
-- name: Install Distribution - Release - Machine Prerequisites
+- name: Inst - Prerequisites
   package:
     name: "{{ packages | flatten(levels=1) }}"
     state: latest
   tags:
-    - install-dependencies
+    - common-inst-prerequisites
 
 - name: Install CSIT PIP requirements
   pip:
diff --git a/resources/tools/testbed-setup/ansible/roles/consul/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/consul/tasks/main.yaml
index d4be90b40b..99ac52da44 100644
--- a/resources/tools/testbed-setup/ansible/roles/consul/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/consul/tasks/main.yaml
@@ -1,12 +1,14 @@
 ---
 # file: roles/consul/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - consul-inst-prerequisites
 
 - name: Inst - Prerequisites
   package:
diff --git a/resources/tools/testbed-setup/ansible/roles/dpdk/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/dpdk/tasks/main.yaml
index a38a9ee429..27a5c2c9e3 100644
--- a/resources/tools/testbed-setup/ansible/roles/dpdk/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/dpdk/tasks/main.yaml
@@ -1,21 +1,23 @@
 ---
 # file: roles/dpdk/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - dpdk-inst-prerequisites
 
-- name: DPDK - Install Distribution - Release - Machine Prerequisites
+- name: Inst - Prerequisites
   package:
     name: "{{ packages | flatten(levels=1) }}"
     state: latest
   tags:
-    - install-dependencies
+    - dpdk-inst-prerequisites
 
-- name: DPDK - Download Release Archive
+- name: Download Release Archive
   get_url:
     url: "{{ dpdk_url }}/dpdk-{{ item }}.tar.xz"
     dest: "{{ dpdk_target_dir }}/dpdk-{{ item }}.tar.xz"
@@ -25,7 +27,7 @@
   tags:
     - install-dpdk
 
-- name: DPDK - Extract Release Archive
+- name: Extract Release Archive
   unarchive:
     remote_src: true
     src: "{{ dpdk_target_dir }}/dpdk-{{ item }}.tar.xz"
@@ -37,7 +39,7 @@
   tags:
     - install-dpdk
 
-- name: DPDK - Build igb_uio by default
+- name: Build igb_uio by default
   lineinfile:
     dest: "{{ dpdk_target_dir }}/dpdk-{{ item }}/config/common_base"
     regexp: "^CONFIG_RTE_EAL_IGB_UIO"
@@ -48,7 +50,7 @@
   tags:
     - install-dpdk
 
-- name: DPDK - Compile Release I
+- name: Compile Release I
   become: yes
   command: "make install T={{ dpdk_build_targets[item][ansible_machine] }} DESTDIR={{ dpdk_target_dir }}/dpdk-{{ item }} chdir={{ dpdk_target_dir }}/dpdk-{{ item }}"
   loop: "{{ dpdk_version }}"
@@ -57,7 +59,7 @@
   tags:
     - install-dpdk
 
-- name: DPDK - Link igb_uio Module
+- name: Link igb_uio Module
   shell: "ln -fs {{ dpdk_target_dir }}/dpdk-{{ item }}/{{ dpdk_build_targets[item][ansible_machine] }}/kmod/igb_uio.ko /lib/modules/`uname -r`/igb_uio.ko && depmod -a"
   ignore_errors: "yes"
   loop: "{{ dpdk_version }}"
diff --git a/resources/tools/testbed-setup/ansible/roles/iperf/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/iperf/tasks/main.yaml
index 00940e4958..853a00cb86 100644
--- a/resources/tools/testbed-setup/ansible/roles/iperf/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/iperf/tasks/main.yaml
@@ -1,21 +1,23 @@
 ---
 # file: roles/iperf/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - iperf-inst-prerequisites
 
-- name: iPerf - Install Distribution - Release - Machine Prerequisites
+- name: Inst - Prerequisites
   package:
     name: "{{ packages | flatten(levels=1) }}"
     state: latest
   tags:
-    - install-dependencies
+    - iperf-inst-prerequisites
 
-- name: iPerf - Get Release Archive
+- name: Get Release Archive
   get_url:
     url: "https://downloads.es.net/pub/iperf/iperf-{{ item }}.tar.gz"
     dest: "{{ iperf_target_dir }}/iperf-{{ item }}.tar.gz"
@@ -25,7 +27,7 @@
   tags:
     - install-iperf
 
-- name: iPerf - Extract Release Archive
+- name: Extract Release Archive
   unarchive:
     remote_src: true
     src: "{{ iperf_target_dir }}/iperf-{{ item }}.tar.gz"
@@ -35,7 +37,7 @@
   tags:
     - install-iperf
 
-- name: iPerf - Compile Release I
+- name: Compile Release I
   command: "./configure"
   args:
       chdir: "{{ iperf_target_dir }}/iperf-{{ item }}/"
@@ -43,7 +45,7 @@
   tags:
     - install-iperf
 
-- name: iPerf - Compile Release II
+- name: Compile Release II
   command: "make"
   args:
       chdir: "{{ iperf_target_dir }}/iperf-{{ item }}/"
@@ -51,7 +53,7 @@
   tags:
     - install-iperf
 
-- name: iPerf - Compile Release III
+- name: Compile Release III
   command: "make install"
   args:
       chdir: "{{ iperf_target_dir }}/iperf-{{ item }}/"
diff --git a/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml
index f677836702..1b81215a73 100644
--- a/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml
@@ -1,21 +1,23 @@
 ---
 # file: roles/mellanox/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - mellanox-inst-prerequisites
 
-- name: Mellanox Install - Install Distribution - Release - Machine Prerequisites
+- name: Inst - Prerequisites
   package:
     name: "{{ packages | flatten(levels=1) }}"
     state: latest
   tags:
-    - install-dependencies
+    - mellanox-inst-prerequisites
 
-- name: Mellanox Install - Check Presence of Mellanox Hardware
+- name: Check Presence of Mellanox Hardware
   shell: "lspci | grep Mellanox | awk '{print $1}'"
   register: mellanox_pcis
   failed_when: no
@@ -23,7 +25,7 @@
   tags:
     - install-mellanox
 
-- name: Mellanox Install - Get OFED
+- name: Get OFED
   get_url:
     url: "http://content.mellanox.com/ofed/MLNX_OFED-{{ mellanox_version }}/MLNX_OFED_LINUX-{{ mellanox_version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz"
     dest: "/opt/MLNX_OFED_LINUX-{{ mellanox_version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz"
@@ -32,7 +34,7 @@
   tags:
     - install-mellanox
 
-- name: Mellanox Install - Extract OFED
+- name: Extract OFED
   unarchive:
     remote_src: true
     src: "/opt/MLNX_OFED_LINUX-{{ mellanox_version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}.tgz"
@@ -43,7 +45,7 @@
   tags:
     - install-mellanox
 
-- name: Mellanox Install - Install OFED
+- name: Install OFED
   command: "./mlnxofedinstall --with-mft --dpdk --force --upstream-libs"
   args:
     chdir: "/opt/MLNX_OFED_LINUX-{{ mellanox_version }}-{{ ansible_distribution|lower }}{{ ansible_distribution_version }}-{{ ansible_machine }}"
@@ -51,13 +53,13 @@
   tags:
     - install-mellanox
 
-- name: Mellanox Install - Switch Infiniband to Ethernet
+- name: Switch Infiniband to Ethernet
   command: "mlxconfig --yes --dev {{ item }} set LINK_TYPE_P1=2 LINK_TYPE_P2=2"
   with_items: "{{ mellanox_pcis.stdout_lines }}"
   tags:
     - install-mellanox
 
-- name: Mellanox Install - FIX qemu-system removal
+- name: FIX qemu-system removal
   package:
     name: "qemu-system"
     state: latest
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml
index 5b323054d4..54e80513b8 100644
--- a/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml
@@ -1,12 +1,14 @@
 ---
 # file: roles/nomad/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - nomad-inst-prerequisites
 
 - name: Inst - Prerequisites
   package:
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml
index 608d3e2505..3c22892483 100644
--- a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/main.yaml
@@ -1,26 +1,28 @@
 ---
 # file: roles/performance_tuning/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - perf-inst-prerequisites
 
-- name: Performance Tuning - Install Distribution - Release - Machine Prerequisites
+- name: Inst - Machine Prerequisites
   package:
     name: "{{ packages | flatten(levels=1) }}"
     state: latest
   tags:
-    - install-dependencies
+    - perf-inst-prerequisites
 
-- name: Performance Tuning - Distribution - release - machine optimizations
+- name: Distribution - release - machine optimizations
   include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml'
   tags:
     - machine-optimizations
 
-- name: Performance Tuning - Configure {{ ansible_machine }} Kernel Parameters
+- name: Configure {{ ansible_machine }} Kernel Parameters
   lineinfile:
     path: "/etc/default/grub"
     state: "present"
@@ -33,7 +35,7 @@
 
 - meta: flush_handlers
 
-- name: Performance Tuning - Turbo Boost
+- name: Turbo Boost
   import_tasks: turbo_boost.yaml
   when: >
     cpu_microarchitecture == "skylake" or
@@ -41,7 +43,7 @@
   tags:
     - turbo-boost
 
-- name: Performance Tuning - Adjust nr_hugepages
+- name: Adjust nr_hugepages
   # change the minimum size of the hugepage pool.
   # 2G VPP, 4GB per VNF/CNF, 2G reserve
   sysctl:
@@ -53,7 +55,7 @@
   tags:
     - set-sysctl
 
-- name: Performance Tuning - Adjust max_map_count
+- name: Adjust max_map_count
   # this file contains the maximum number of memory map areas a process
   # may have. memory map areas are used as a side-effect of calling
   # malloc, directly by mmap and mprotect, and also when loading shared
@@ -72,7 +74,7 @@
   tags:
     - set-sysctl
 
-- name: Performance Tuning - Adjust hugetlb_shm_group
+- name: Adjust hugetlb_shm_group
   # hugetlb_shm_group contains group id that is allowed to create sysv
   # shared memory segment using hugetlb page.
   sysctl:
@@ -84,7 +86,7 @@
   tags:
     - set-sysctl
 
-- name: Performance Tuning - Adjust swappiness
+- name: Adjust swappiness
   # this control is used to define how aggressive the kernel will swap
   # memory pages.  higher values will increase agressiveness, lower values
   # decrease the amount of swap.  a value of 0 instructs the kernel not to
@@ -99,7 +101,7 @@
   tags:
     - set-sysctl
 
-- name: Performance Tuning - Adjust shmmax
+- name: Adjust shmmax
   # shared memory max must be greator or equal to the total size of hugepages.
   # for 2mb pages, totalhugepagesize = vm.nr_hugepages * 2 * 1024 * 1024
   # if the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax)
@@ -114,7 +116,7 @@
   tags:
     - set-sysctl
 
-- name: Performance Tuning - Adjust watchdog_cpumask
+- name: Adjust watchdog_cpumask
   # this value can be used to control on which cpus the watchdog may run.
   # the default cpumask is all possible cores, but if no_hz_full is
   # enabled in the kernel config, and cores are specified with the
@@ -134,7 +136,7 @@
   tags:
     - set-sysctl
 
-- name: Performance Tuning - Adjust randomize_va_space
+- name: Adjust randomize_va_space
   # this option can be used to select the type of process address
   # space randomization that is used in the system, for architectures
   # that support this feature.
@@ -150,7 +152,7 @@
   tags:
     - set-sysctl
 
-- name: Performance Tuning - Copy Cpufrequtils File
+- name: Copy Cpufrequtils File
   copy:
     src: "files/cpufrequtils"
     dest: "/etc/default/cpufrequtils"
@@ -160,7 +162,7 @@
   tags:
     - copy-cpufrequtils
 
-- name: Performance Tuning - Copy Irqbalance File
+- name: Copy Irqbalance File
   template:
     src: "files/irqbalance"
     dest: "/etc/default/irqbalance"
@@ -172,14 +174,14 @@
   tags:
     - copy-irqbalance
 
-- name: Performance Tuning - Set Ondemand Service To Disable
+- name: Set Ondemand Service To Disable
   service:
     name: "ondemand"
     enabled: "no"
   tags:
     - set-ondemand
 
-- name: Performance Tuning - Load Kernel Modules By Default
+- name: Load Kernel Modules By Default
   lineinfile:
     path: "/etc/modules"
     state: "present"
diff --git a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml
index 23cca0566e..11f0e326d9 100644
--- a/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/performance_tuning/tasks/turbo_boost.yaml
@@ -1,22 +1,24 @@
 ---
 # file: roles/performance_tuning/tasks/turbo_boost.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - turbo-inst-prerequisites
 
-- name: Turbo Boost - Install msr-tools
+- name: Install msr-tools
   package:
     name:
       - "msr-tools"
     state: latest
   tags:
-    - turbo-boost
+    - turbo-inst-prerequisites
 
-- name: Turbo Boost - Load msr By Default
+- name: Load msr By Default
   lineinfile:
     path: "/etc/modules"
     state: "present"
@@ -24,7 +26,7 @@
   tags:
     - turbo-boost
 
-- name: Turbo Boost - Custom Startup Service Hook
+- name: Custom Startup Service Hook
   copy:
     src: "files/disable-turbo-boost.service"
     dest: "/etc/systemd/system/disable-turbo-boost.service"
@@ -34,7 +36,7 @@
   tags:
     - turbo-boost
 
-- name: Turbo Boost - Custom Startup Service Hook Enable
+- name: Custom Startup Service Hook Enable
   service:
     name: "disable-turbo-boost"
     enabled: yes
diff --git a/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/defaults/main.yaml
new file mode 100644
index 0000000000..eb2b94cb26
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/defaults/main.yaml
@@ -0,0 +1,17 @@
+---
+# file: roles/prometheus_exporter/defaults/main.yaml
+
+# Inst - Exporters.
+ne_packages: "{{ ne_packages_by_distro[ansible_distribution | lower][ansible_machine] }}"
+
+ne_packages_by_distro:
+  ubuntu:
+    aarch64: "http://ports.ubuntu.com/pool/universe/p/prometheus-node-exporter/prometheus-node-exporter_1.0.1+ds-1_arm64.deb"
+    x86_64: "http://archive.ubuntu.com/ubuntu/pool/universe/p/prometheus-node-exporter/prometheus-node-exporter_1.0.1+ds-1_amd64.deb"
+
+be_packages: "{{ be_packages_by_distro[ansible_distribution | lower][ansible_machine] }}"
+
+be_packages_by_distro:
+  ubuntu:
+    aarch64: "http://ports.ubuntu.com/pool/universe/p/prometheus-blackbox-exporter/prometheus-blackbox-exporter_0.17.0+ds-1_arm64.deb"
+    x86_64: "http://archive.ubuntu.com/ubuntu/pool/universe/p/prometheus-blackbox-exporter/prometheus-blackbox-exporter_0.17.0+ds-1_amd64.deb"
diff --git a/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/files/blackbox.yml b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/files/blackbox.yml
new file mode 100644
index 0000000000..f61c26e1a8
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/files/blackbox.yml
@@ -0,0 +1,25 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      no_follow_redirects: false
+      fail_if_ssl: false
+      fail_if_not_ssl: true
+      tls_config:
+        insecure_skip_verify: false
+      preferred_ip_protocol: "ip4"
+  icmp_v4:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      preferred_ip_protocol: "ip4"
+  dns_udp:
+    prober: dns
+    timeout: 5s
+    dns:
+      query_name: "jenkins.fd.io"
+      query_type: "A"
+      valid_rcodes:
+      - NOERROR
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/handlers/main.yaml
new file mode 100644
index 0000000000..9c374eaa61
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/handlers/main.yaml
@@ -0,0 +1,16 @@
+---
+# file roles/prometheus_exporter/handlers/main.yaml
+
+- name: Restart Prometheus Node Exporter
+  systemd:
+    daemon_reload: true
+    enabled: true
+    name: "prometheus-node-exporter"
+    state: "restarted"
+
+- name: Restart Prometheus Blackbox Exporter
+  systemd:
+    daemon_reload: true
+    enabled: true
+    name: "prometheus-blackbox-exporter"
+    state: "restarted"
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/tasks/main.yaml
new file mode 100644
index 0000000000..b38215c4a2
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/tasks/main.yaml
@@ -0,0 +1,15 @@
+---
+# file: roles/prometheus_exporter/tasks/main.yaml
+
+- include_tasks: "{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml"
+  tags:
+    - prometheus-inst
+
+- name: Conf - Prometheus Blackbox Exporter
+  copy:
+   src: 'files/blackbox.yml'
+   dest: '/etc/prometheus/blackbox.yml'
+  notify:
+    - "Restart Prometheus Blackbox Exporter"
+  tags:
+    - prometheus-conf-blackbox-exporter
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/tasks/ubuntu_bionic.yaml
new file mode 100644
index 0000000000..566753e272
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/prometheus_exporter/tasks/ubuntu_bionic.yaml
@@ -0,0 +1,33 @@
+---
+# file: roles/prometheus_exporter/tasks/ubuntu_bionic.yaml
+
+- name: Inst - Update Package Cache (APT)
+  apt:
+    update_cache: yes
+    cache_valid_time: 3600
+  tags:
+    - prometheus-inst-prerequisites
+
+- name: Inst - Prerequisites
+  package:
+    name: "init-system-helpers"
+    default_release: "bionic-backports"
+    state: latest
+  tags:
+    - prometheus-inst-prerequisites
+
+- name: Inst - Prometheus Node Exporter
+  apt:
+    deb: "{{ ne_packages }}"
+  notify:
+    - "Restart Prometheus Node Exporter"
+  tags:
+    - prometheus-inst-node-exporter
+
+- name: Inst - Prometheus Blackbox Exporter
+  apt:
+    deb: "{{ be_packages }}"
+  notify:
+    - "Restart Prometheus Blackbox Exporter"
+  tags:
+    - prometheus-inst-blackbox-exporter
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/trex/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/trex/tasks/main.yaml
index 1e442f5a58..96a3ba5f03 100644
--- a/resources/tools/testbed-setup/ansible/roles/trex/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/trex/tasks/main.yaml
@@ -1,19 +1,21 @@
 ---
 # file: roles/trex/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - trex-inst-prerequisites
 
-- name: Install Distribution - Release - Machine Prerequisites
+- name: Inst - Prerequisites
   package:
     name: "{{ packages | flatten(levels=1) }}"
     state: latest
   tags:
-    - install-dependencies
+    - trex-inst-prerequisites
 
 - name: Deploy Multiple T-Rex Versions
   include_tasks: deploy_block.yaml
diff --git a/resources/tools/testbed-setup/ansible/roles/vpp/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/vpp/tasks/main.yaml
index 06930b2479..6011917de2 100644
--- a/resources/tools/testbed-setup/ansible/roles/vpp/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/vpp/tasks/main.yaml
@@ -1,21 +1,23 @@
 ---
-# file: roles/sut/tasks/main.yaml
+# file: roles/vpp/tasks/main.yaml
 
-- Name: Update package cache (apt)
+- name: Inst - Update Package Cache (APT)
   apt:
     update_cache: yes
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
+  tags:
+    - vpp-inst-prerequisites
 
-- name: SUT - Install Distribution - Release - Machine Prerequisites
+- name: Inst - Prerequisites
   package:
     name: "{{ packages | flatten(levels=1) }}"
     state: latest
   tags:
-    - install-dependencies
+    - vpp-inst-prerequisites
 
-- name: SUT - Install VPP 19.08 PIP requirements
+- name: Inst - VPP 19.08 PIP requirements
   pip:
     name:
       - "aenum==2.1.2"
@@ -23,7 +25,7 @@
   tags:
     - install-pip
 
-- name: SUT - Copy 80-vpp.conf
+- name: Copy 80-vpp.conf
   file:
     src: "/dev/null"
     dest: "/etc/sysctl.d/80-vpp.conf"
diff --git a/resources/tools/testbed-setup/ansible/vpp_device.yaml b/resources/tools/testbed-setup/ansible/vpp_device.yaml
index ac42b8cafe..7dc3614bbc 100644
--- a/resources/tools/testbed-setup/ansible/vpp_device.yaml
+++ b/resources/tools/testbed-setup/ansible/vpp_device.yaml
@@ -18,6 +18,10 @@
       tags: nomad
     - role: consul
       tags: consul
+    - role: prometheus_exporter
+      tags: prometheus_exporter
+    - role: cadvisor
+      tags: cadvisor
     - role: vpp_device
       tags: vpp_device
     - role: kernel_vm