Python3: PIP requirement

- Install via PIP3
- Use virtualenv Python3 interpret

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: Iad22b344b418a1d91774d23d5ea17bfd48d394b3

14 files changed, 150 insertions, 92 deletions

diff --git a/pylint.cfg b/pylint.cfg
index 69668ed79c..5ae4ed551b 100644
--- a/pylint.cfg
+++ b/pylint.cfg
@@ -12,7 +12,7 @@ profile=no
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=CVS
+ignore=CVS, conf.py
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -282,3 +282,10 @@ max-public-methods=60
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
 overgeneral-exceptions=Exception
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=fstr
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 3fa2a9d879..24263c7aae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,57 +1,79 @@
 # DISCLAIMER: BEFORE EDITING THIS FILE!
 #
-# This file has two closely related consequences.
-# The common part is that this file defines
-# the content of virtual Python environment
-# used when Robot tests are running.
-# The difference is in which machine the environment is created on.
-# For the Jenkins executor machines,
-# editing this file has no surprising consequences.
+# This file has two closely related consequences. The common part is that this
+# file defines the content of virtual Python environment used when Robot tests
+# are running.
 #
-# But the environment is also being created
-# on virtual machines created by VIRL, which do have specific restrictions.
-# Complete package and setting snapshot (for both VIRL and Physical Testbed),
-# with network being limited in case of VIRL to local only.
-# This implies pip currently being set to site-packages only
-# for both VIRL and Physical Testbed.
-# So if the "virl image" applied on the machine
-# does not reflect the edits, some installations might fail.
-# Even if the failure may not directly impact execution,
-# this leads into inconsistency between installed environment
-# and the code base which may lead into fatal error or uncaught exceptions.
+# But the virtual environment is also being created on containerized
+# environment created by either Nomad cluster (Jenkins, vpp_device) or during
+# execution on SUT machines, which might have specific restrictions.
 #
-# The "virl image" is built from files such as
-# resources/tools/disk-image-builder/ubuntu/lists/ubuntu-16.04.1_2017-10-21_2.0/pip-requirements.txt
-# but the build is not automated,
-# the new "virl image" has to be built and applied manually
-# before your edit can pass.
-# This also means, incompatible virl images
-# would make existing tests fail until your edit is merged.
+# Complete package and setting snapshot (for all container images),
+# with network being limited to local only. This implies pip currently being
+# set to site-packages only. So if the "container image" applied on the machine
+# does not reflect the edits, some installations might fail.
+# Even if the failure may not directly impact execution, this leads into
+# inconsistency between installed environment and the code base which may lead
+# into fatal error or uncaught exceptions.
 #
-# TODO: Automate the virtualenv download and distribution
-#   on all types of testbeds prior the test execution.
-# TODO: Figure out a way to verify edits to this file automatically.
-
-# TODO: Split into directly needed packages and their dependencies.
-aenum==2.1.2
-docopt==0.6.2  # for pykwalify
-ecdsa==0.13
-enum34==1.1.2
-ipaddress==1.0.16
-paramiko==1.16.0
-pexpect==4.6.0
+# CSIT Core dependencies
+ecdsa==0.13.3
+paramiko==2.6.0
 pycrypto==2.6.1
-pykwalify==1.5.0
-pypcap==1.1.5
-python-dateutil==2.4.2  # for pykwalify
-PyYAML==3.11
-requests==2.9.1
-robotframework==2.9.2
-scapy==2.3.1
-scp==0.10.2
-six==1.12.0  # for python-dateutil
+pypcap==1.2.3
+PyYAML==5.1.1
+requests==2.22.0
+robotframework==3.1.2
+scapy==2.4.3
+scp==0.13.2
 
-# The following is only there for PLRsearch.
+# Bootstraping dependencies
+ansible==2.7.8
+
+# PLRsearch dependencies
 dill==0.2.8.2
-numpy==1.14.5
+numpy==1.17.3
 scipy==1.1.0
+
+# PAL/DOC dependencies
+hdrhistogram==0.6.1
+pandas==0.25.3
+plotly==4.1.1
+PTable==0.9.2
+Sphinx==2.2.1
+sphinx-rtd-theme==0.4.0
+sphinxcontrib-programoutput==0.15
+sphinxcontrib-robotdoc==0.11.0
+
+# PIP freeze dependencies
+alabaster==0.7.12
+Babel==2.7.0
+bcrypt==3.1.7
+certifi==2019.9.11
+cffi==1.13.2
+chardet==3.0.4
+cryptography==2.8
+docutils==0.15.2
+future==0.18.2
+idna==2.8
+imagesize==1.1.0
+Jinja2==2.10.3
+MarkupSafe==1.1.1
+packaging==19.2
+pbr==5.4.3
+pycparser==2.19
+Pygments==2.4.2
+PyNaCl==1.3.0
+pyparsing==2.4.4
+python-dateutil==2.8.1
+pytz==2019.3
+retrying==1.3.3
+six==1.13.0
+snowballstemmer==2.0.0
+sphinxcontrib-applehelp==1.0.1
+sphinxcontrib-devhelp==1.0.1
+sphinxcontrib-htmlhelp==1.0.2
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.2
+sphinxcontrib-serializinghtml==1.1.3
+urllib3==1.25.6
diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index 1ef7c93da3..3a7a0fbf7d 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -113,16 +113,16 @@ function activate_virtualenv () {
     env_dir="${root_path}/env"
     req_path=${2-$CSIT_DIR/requirements.txt}
     rm -rf "${env_dir}" || die "Failed to clean previous virtualenv."
-    pip install --upgrade virtualenv || {
+    pip3 install --upgrade virtualenv || {
         die "Virtualenv package install failed."
     }
-    virtualenv "${env_dir}" || {
-        die "Virtualenv creation failed."
+    virtualenv -p $(which python3) "${env_dir}" || {
+        die "Virtualenv creation for $(which python) failed."
     }
     set +u
     source "${env_dir}/bin/activate" || die "Virtualenv activation failed."
     set -u
-    pip install --upgrade -r "${req_path}" || {
+    pip3 install --upgrade -r "${req_path}" || {
         die "Requirements installation failed."
     }
     # Most CSIT Python scripts assume PYTHONPATH is set and exported.
@@ -637,7 +637,8 @@ function run_pybot () {
 
     pushd "${CSIT_DIR}" || die "Change directory operation failed."
     set +e
-    pybot "${all_options[@]}" "${GENERATED_DIR}/tests/"
+    python -V
+    robot "${all_options[@]}" "${GENERATED_DIR}/tests/"
     PYBOT_EXIT_STATUS="$?"
     set -e
 
diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py
index 760d958c54..3f74057832 100644
--- a/resources/libraries/python/SetupFramework.py
+++ b/resources/libraries/python/SetupFramework.py
@@ -111,8 +111,9 @@ def create_env_directory_at_node(node):
                    .format(host))
     exec_cmd_no_error(
         node, 'cd {0} && rm -rf env'
-        ' && virtualenv --system-site-packages --never-download env'
-        ' && source env/bin/activate && pip install -r requirements.txt'
+        ' && virtualenv -p $(which python3) '
+        '--system-site-packages --never-download env'
+        ' && source env/bin/activate && pip3 install -r requirements.txt'
         .format(con.REMOTE_FW_DIR), timeout=100, include_reason=True,
         message="Failed install at node {host}".format(host=host))
     logger.console('Virtualenv setup on {0} done.'.format(host))
diff --git a/resources/tools/doc_gen/run_doc.sh b/resources/tools/doc_gen/run_doc.sh
index 523b4ae505..36535df031 100755
--- a/resources/tools/doc_gen/run_doc.sh
+++ b/resources/tools/doc_gen/run_doc.sh
@@ -33,8 +33,6 @@ virtualenv ${WORKING_DIR}/env
 
 # Install CSIT requirements:
 pip install -r ../../../requirements.txt
-# Install Sphinx:
-pip install -r ${WORKING_DIR}/requirements.txt
 
 export PYTHONPATH=`pwd`
 
diff --git a/resources/tools/doc_gen/src/requirements.txt b/resources/tools/doc_gen/src/requirements.txt
deleted file mode 100644
index 2cf7cd9687..0000000000
--- a/resources/tools/doc_gen/src/requirements.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-Sphinx==1.5.5
-sphinxcontrib_robotdoc
-sphinx-rtd-theme
\ No newline at end of file
diff --git a/resources/tools/presentation/run_cpta.sh b/resources/tools/presentation/run_cpta.sh
index 954f17d53c..a60c1cc7a9 100755
--- a/resources/tools/presentation/run_cpta.sh
+++ b/resources/tools/presentation/run_cpta.sh
@@ -16,11 +16,11 @@ sudo apt-get -y install libxml2 libxml2-dev libxslt-dev build-essential \
 mkdir ${DIR[WORKING]}
 
 # Create virtual environment
-virtualenv ${DIR[WORKING]}/env
-. ${DIR[WORKING]}/env/bin/activate
+virtualenv -p $(which python3) ${DIR[WORKING]}/env
+source ${DIR[WORKING]}/env/bin/activate
 
 # Install python dependencies:
-pip install -r requirements.txt
+pip3 install -r requirements.txt
 
 export PYTHONPATH=`pwd`
 
diff --git a/resources/tools/presentation/run_report.sh b/resources/tools/presentation/run_report.sh
index 24b6c21a70..56c671a8bb 100755
--- a/resources/tools/presentation/run_report.sh
+++ b/resources/tools/presentation/run_report.sh
@@ -27,11 +27,11 @@ fi
 mkdir ${DIR[WORKING]}
 
 # Create virtual environment
-virtualenv ${DIR[WORKING]}/env
-. ${DIR[WORKING]}/env/bin/activate
+virtualenv -p $(which python3) ${DIR[WORKING]}/env
+source ${DIR[WORKING]}/env/bin/activate
 
 # Install python dependencies:
-pip install -r requirements.txt
+pip3 install -r requirements.txt
 
 export PYTHONPATH=`pwd`
 
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
index f53584a939..f2e76a729d 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
@@ -1,8 +1,8 @@
 ---
 # file: lf_inventory/group_vars/all.yaml
 
-# General variables
-ansible_python_interpreter: '/usr/bin/python2.7'
+# Ansible interpreter (for PIP)
+ansible_python_interpreter: 'python3'
 # provision via cobbler
 provision_enabled: False
 # name_servers_search is used in /etc/hosts file on target machine.
diff --git a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml
index 484e6ddb4b..d02746cb4f 100644
--- a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml
@@ -1,8 +1,8 @@
 ---
 # file: group_vars/all.yaml
 
-# General variables
-ansible_python_interpreter: '/usr/bin/python2.7'
+# Ansible interpreter (for PIP)
+ansible_python_interpreter: 'python3'
 # provision via cobbler
 provision_enabled: True
 # name_servers_search is used in /etc/hosts file on target machine.
diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template
index cf2fbdfe34..965562ae0a 100644
--- a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template
+++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template
@@ -21,7 +21,7 @@ option pxe-system-type code 93 = unsigned integer 16;
 subnet 10.30.51.0 netmask 255.255.255.0 {
      option routers             10.30.51.1;
      option domain-name         "linuxfoundation.org";
-     option domain-name-servers 199.204.44.24, 199.204.47.54;
+     option domain-name-servers 1.1.1.1, 8.8.8.8;
      option subnet-mask         255.255.255.0;
      range dynamic-bootp        10.30.51.2 10.30.51.254;
      default-lease-time         600;
diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed
index 250701232c..3827562c06 100644
--- a/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed
+++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed
@@ -114,7 +114,7 @@ d-i user-setup/encrypt-home boolean false
 ### Package selection
 tasksel tasksel/first multiselect ubuntu-server
 # Individual additional packages to install
-d-i pkgsel/include string openssh-server python2.7
+d-i pkgsel/include string openssh-server python2.7 python3.6
 # Whether to upgrade packages after debootstrap.
 # Allowed values: none, safe-upgrade, full-upgrade
 #d-i pkgsel/upgrade select none
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index 3e7e5a00fa..84358bf863 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -70,25 +70,57 @@
 - name: Install CSIT PIP requirements
   pip:
     name:
-      - 'docopt==0.6.2'
-      - 'ecdsa==0.13'
-      - 'enum34==1.1.2'
-      - 'ipaddress==1.0.16'
-      - 'paramiko==1.16.0'
-      - 'pexpect==4.6.0'
+      - 'ecdsa==0.13.3'
+      - 'paramiko==2.6.0'
       - 'pycrypto==2.6.1'
-      - 'pykwalify==1.5.0'
-      - 'pypcap==1.1.5'
-      - 'python-dateutil==2.4.2'
-      - 'PyYAML==3.11'
-      - 'requests==2.9.1'
-      - 'robotframework==2.9.2'
-      - 'scapy==2.3.1'
-      - 'scp==0.10.2'
-      - 'six==1.12.0'
+      - 'pypcap==1.2.3'
+      - 'PyYAML==5.1.1'
+      - 'requests==2.22.0'
+      - 'robotframework==3.1.2'
+      - 'scapy==2.4.3'
+      - 'scp==0.13.2'
+      - 'ansible==2.7.8'
       - 'dill==0.2.8.2'
-      - 'numpy==1.14.5'
-      - 'aenum==2.1.2'
+      - 'numpy==1.17.3'
+      - 'hdrhistogram==0.6.1'
+      - 'pandas==0.25.3'
+      - 'plotly==4.1.1'
+      - 'PTable==0.9.2'
+      - 'Sphinx==2.2.1'
+      - 'sphinx-rtd-theme==0.4.0'
+      - 'sphinxcontrib-programoutput==0.15'
+      - 'sphinxcontrib-robotdoc==0.11.0'
+      - 'alabaster==0.7.12'
+      - 'Babel==2.7.0'
+      - 'bcrypt==3.1.7'
+      - 'certifi==2019.9.11'
+      - 'cffi==1.13.2'
+      - 'chardet==3.0.4'
+      - 'cryptography==2.8'
+      - 'docutils==0.15.2'
+      - 'future==0.18.2'
+      - 'idna==2.8'
+      - 'imagesize==1.1.0'
+      - 'Jinja2==2.10.3'
+      - 'MarkupSafe==1.1.1'
+      - 'packaging==19.2'
+      - 'pbr==5.4.3'
+      - 'pycparser==2.19'
+      - 'Pygments==2.4.2'
+      - 'PyNaCl==1.3.0'
+      - 'pyparsing==2.4.4'
+      - 'python-dateutil==2.8.1'
+      - 'pytz==2019.3'
+      - 'retrying==1.3.3'
+      - 'six==1.13.0'
+      - 'snowballstemmer==2.0.0'
+      - 'sphinxcontrib-applehelp==1.0.1'
+      - 'sphinxcontrib-devhelp==1.0.1'
+      - 'sphinxcontrib-htmlhelp==1.0.2'
+      - 'sphinxcontrib-jsmath==1.0.1'
+      - 'sphinxcontrib-qthelp==1.0.2'
+      - 'sphinxcontrib-serializinghtml==1.1.3'
+      - 'urllib3==1.25.6'
   tags: install-pip
 
 - name: Install CSIT PIP requirements - SciPy workaround
diff --git a/tox.ini b/tox.ini
index b235bf3ab8..4617e73e54 100644
--- a/tox.ini
+++ b/tox.ini
@@ -36,9 +36,9 @@ checker_dir = ./resources/libraries/bash/entry/check
 # TODO: Tox prints various warnings. Figure them out and fix them.
 
 [testenv:pylint]
-basepython = python2
+basepython = python3
 deps =
-    pylint==1.5.4
+    pylint==2.4.3
     -r ./requirements.txt
 whitelist_externals = /bin/bash
 setenv = PYTHONPATH = {toxinidir}