Add PAPI dependencies

+ Various small structure optmizations

Change-Id: I1aac9353f72436de53797fc734bef498905c8189
Signed-off-by: Peter Mikus <pmikus@cisco.com>

12 files changed, 57 insertions, 102 deletions

diff --git a/resources/tools/disk-image-builder/csit-sut-dcr/ubuntu_1804_amd64/Dockerfile b/resources/tools/disk-image-builder/csit-sut-dcr/ubuntu_1804_amd64/Dockerfile
index c7aa3363cf..a681c21e4f 100644
--- a/resources/tools/disk-image-builder/csit-sut-dcr/ubuntu_1804_amd64/Dockerfile
+++ b/resources/tools/disk-image-builder/csit-sut-dcr/ubuntu_1804_amd64/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -14,9 +14,8 @@
 FROM ubuntu:bionic-20180821
 
 MAINTAINER csit-dev <csit-dev@lists.fd.io>
-LABEL Description="CSIT vpp-device ubuntu 18.04 baseline image"
-LABEL Vendor="cisco.com"
-LABEL Version="0.4"
+LABEL Description="CSIT vpp-device ubuntu 18.04 baseline image (x86)"
+LABEL Version="0.6"
 
 # Setup the environment
 ENV DEBIAN_FRONTEND=noninteractive
@@ -82,6 +81,7 @@ RUN apt-get -q update \
         libnuma1 \
         python-cffi \
         python-enum34 \
+        python3-cffi \
  && rm -rf /var/lib/apt/lists/*
 
 # Configure locales
@@ -97,7 +97,7 @@ RUN mkdir -p /tmp/dumps \
  && mkdir -p /var/cache/vpp/python \
  && mkdir -p /var/run/sshd
 
-# PIP pre-cache
+# CSIT PIP pre-cache
 RUN pip install \
         docopt==0.6.2 \
         ecdsa==0.13 \
@@ -119,6 +119,10 @@ RUN pip install \
         numpy==1.14.5 \
         scipy==1.1.0
 
+# VPP PIP pre-cache
+RUN pip install \
+        aenum
+
 # SSH settings
 RUN echo 'root:Csit1234' | chpasswd \
  && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \
diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources-arm.list b/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources_bionic_aarch64.list
index 08da1ecf96..08da1ecf96 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources-arm.list
+++ b/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources_bionic_aarch64.list
diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources.list b/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources_bionic_x86_64.list
index e8ccadfc64..e8ccadfc64 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources.list
+++ b/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources_bionic_x86_64.list
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index 6844358eca..d7355883e0 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -9,9 +9,8 @@
   with_dict: "{{ proxy_env }}"
   when: proxy_env is defined
 
-- name: Ubuntu specific
-  import_tasks: ubuntu.yaml
-  when: ansible_distribution|lower == 'ubuntu'
+- name: Install distribution - release - machine prerequisites
+  include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml'
 
 - name: Set hostname
   hostname:
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_bionic.yaml
index ae606cd494..d80eb5edac 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_bionic.yaml
@@ -1,24 +1,17 @@
 ---
-# file: roles/common/tasks/ubuntu.yaml
+# file: roles/common/tasks/ubuntu_bionic.yaml
 
-- name: Copy aarch64 apt sources file
+- name: Copy apt sources file
   template:
-    src: 'files/apt-sources-arm.list'
+    src: 'files/apt-sources_{{ ansible_distribution_release }}_{{ ansible_machine }}.list'
     dest: '/etc/apt/sources.list'
   tags: copy-apt-sources
-  when: ansible_machine == 'aarch64'
-
-- name: Copy x86_64 apt sources file
-  template:
-    src: 'files/apt-sources.list'
-    dest: '/etc/apt/sources.list'
-  tags: copy-apt-sources
-  when: ansible_machine == 'x86_64'
 
 - name: Install CSIT dependencies
   apt:
     name: '{{ item }}'
     state: 'present'
+    cache_valid_time: 3600
     install_recommends: False
   with_items:
     - 'python-apt'
diff --git a/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml
index f5a2dd86ad..90990b6c3b 100644
--- a/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml
@@ -1,9 +1,8 @@
 ---
 # file: roles/sut/tasks/main.yaml
 
-- name: Ubuntu specific
-  import_tasks: ubuntu.yaml
-  when: ansible_distribution|lower == 'ubuntu'
+- name: Install distribution - release - machine prerequisites
+  include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml'
 
 - name: Taishan aarch64 specific
   import_tasks: taishan_workaround.yaml
diff --git a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml
index 69ca52916e..7ed9daed22 100644
--- a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml
@@ -1,5 +1,5 @@
 ---
-# file: roles/sut/tasks/ubuntu.yaml
+# file: roles/sut/tasks/ubuntu_bionic.yaml
 
 - name: Install CSIT dependencies
   apt:
@@ -20,4 +20,7 @@
     - 'openjdk-8-jdk'
     - 'libpixman-1-dev'
     - 'python-cffi'
+    - 'python-cffi-backend'
+    - 'python3-cffi'
+    - 'python3-cffi-backend'
   tags: install-csit-dependencies
diff --git a/resources/tools/testbed-setup/ansible/roles/tg/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg/tasks/main.yaml
index 9fa19b5b0f..f3f5d85aa1 100644
--- a/resources/tools/testbed-setup/ansible/roles/tg/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/tg/tasks/main.yaml
@@ -1,6 +1,5 @@
 ---
 # file: roles/tg/tasks/main.yaml
 
-- name: Ubuntu specific
-  import_tasks: ubuntu.yaml
-  when: ansible_distribution|lower == 'ubuntu'
+- name: Install distribution - release - machine prerequisites
+  include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml'
diff --git a/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu_bionic.yaml
index acb4f9802e..f60aaeec64 100644
--- a/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu_bionic.yaml
@@ -1,5 +1,5 @@
 ---
-# file: roles/tg/tasks/ubuntu.yaml
+# file: roles/tg/tasks/ubuntu_bionic.yaml
 
 - name: Install CSIT dependencies
   apt:
diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/requirements.txt b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/requirements.txt
deleted file mode 100644
index 0ce17e2b47..0000000000
--- a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/requirements.txt
+++ /dev/null
@@ -1,56 +0,0 @@
-# DISCLAIMER: BEFORE EDITING THIS FILE!
-#
-# This file has two closely related consequences.
-# The common part is that this file defines
-# the content of virtual Python environment
-# used when Robot tests are running.
-# The difference is in which machine the environment is created on.
-# For the Jenkins executor machines,
-# editing this file has no surprising consequences.
-#
-# But the environment is also being created
-# on virtual machines created by VIRL, which do have specific restrictions.
-# Complete package and setting snapshot (for both VIRL and Physical Testbed),
-# with network being limited in case of VIRL to local only.
-# This implies pip currently being set to site-packages only
-# for both VIRL and Physical Testbed.
-# So if the "virl image" applied on the machine
-# does not reflect the edits, some installations might fail.
-# Even if the failure may not directly impact execution,
-# this leads into inconsistency between installed environment
-# and the code base which may lead into fatal error or uncaught exceptions.
-#
-# The "virl image" is built from files such as
-# resources/tools/disk-image-builder/ubuntu/lists/ubuntu-16.04.1_2017-10-21_2.0/pip-requirements.txt
-# but the build is not automated,
-# the new "virl image" has to be built and applied manually
-# before your edit can pass.
-# This also means, incompatible virl images
-# would make existing tests fail until your edit is merged.
-#
-# TODO: Automate the virtualenv download and distribution
-#   on all types of testbeds prior the test execution.
-# TODO: Figure out a way to verify edits to this file automatically.
-
-# TODO: Split into directly needed packages and their dependencies.
-docopt==0.6.2  # for pykwalify
-ecdsa==0.13
-enum34==1.1.2
-ipaddress==1.0.16
-paramiko==1.16.0
-pexpect==4.6.0
-pycrypto==2.6.1
-pykwalify==1.5.0
-pypcap==1.1.5
-python-dateutil==2.4.2  # for pykwalify
-PyYAML==3.11
-requests==2.9.1
-robotframework==2.9.2
-scapy==2.3.1
-scp==0.10.2
-six==1.12.0  # for python-dateutil
-
-# The following is only there for PLRsearch.
-dill==0.2.8.2
-numpy==1.14.5
-scipy==1.1.0
diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml
index e5da2fc72d..1ad96b26fc 100644
--- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml
@@ -1,22 +1,16 @@
 ---
 # file: roles/tg_sut/tasks/main.yaml
 
-- name: Ubuntu specific
-  import_tasks: ubuntu.yaml
-  when: ansible_distribution|lower == 'ubuntu'
+- name: Install distribution - release - machine prerequisites
+  include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml'
 
-- name: x86 specific
-  import_tasks: x86_64.yaml
-  when: ansible_machine == 'x86_64'
+- name: Machine specifics
+  include_tasks: '{{ ansible_machine }}.yaml'
 
 - name: skylake specific
   import_tasks: skylake.yaml
   when: ("skylake" in groups) and inventory_hostname in groups['skylake']
 
-- name: aarch specific
-  import_tasks: aarch64.yaml
-  when: ansible_machine == 'aarch64'
-
 - name: Copy netplan network config file
   template:
     src: 'files/netplan_config'
@@ -53,21 +47,40 @@
     mode: '0644'
   tags: copy-cpufrequtils
 
-- name: Copy Python requirements file
-  copy:
-    src: 'files/requirements.txt'
-    dest: '/tmp/requirements.txt'
-  tags: copy-pip
-
 - name: Set ondemand service to disable
   service:
     name: 'ondemand'
     enabled: 'no'
   tags: set-ondemand
 
-- name: Install PIP requirements
+- name: Install CSIT PIP requirements
+  pip:
+    name: '{{ item }}'
+  with_items:
+      - 'docopt==0.6.2'
+      - 'ecdsa==0.13'
+      - 'enum34==1.1.2'
+      - 'ipaddress==1.0.16'
+      - 'paramiko==1.16.0'
+      - 'pexpect==4.6.0'
+      - 'pycrypto==2.6.1'
+      - 'pykwalify==1.5.0'
+      - 'pypcap==1.1.5'
+      - 'python-dateutil==2.4.2'
+      - 'PyYAML==3.11'
+      - 'requests==2.9.1'
+      - 'robotframework==2.9.2'
+      - 'scapy==2.3.1'
+      - 'scp==0.10.2'
+      - 'six==1.12.0'
+      - 'dill==0.2.8.2'
+      - 'numpy==1.14.5'
+      - 'scipy==1.1.0'
+  tags: install-pip
+
+- name: Install VPP PIP requirements
   pip:
-    requirements: '/tmp/requirements.txt'
+    name: 'aenum'
   tags: install-pip
 
 - name: Load vfio-pci by default
diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml
index 2f75ae545d..194609722e 100644
--- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml
@@ -1,5 +1,5 @@
 ---
-# file: roles/tg_sut/tasks/ubuntu.yaml
+# file: roles/tg_sut/tasks/ubuntu_bionic.yaml
 
 - name: Install CSIT dependencies
   apt:
@@ -38,6 +38,7 @@
   apt:
     name: 'docker-{{ docker_edition }}={{ docker_apt_package_name }}'
     state: 'present'
+    force: yes
   tags: install-docker
 
 - name: Creates Docker service directory