Python3: PIP requirement

- Install via PIP3
- Use virtualenv Python3 interpret

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: Iad22b344b418a1d91774d23d5ea17bfd48d394b3

11 files changed, 70 insertions, 41 deletions

diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index 1ef7c93da3..3a7a0fbf7d 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -113,16 +113,16 @@ function activate_virtualenv () {
     env_dir="${root_path}/env"
     req_path=${2-$CSIT_DIR/requirements.txt}
     rm -rf "${env_dir}" || die "Failed to clean previous virtualenv."
-    pip install --upgrade virtualenv || {
+    pip3 install --upgrade virtualenv || {
         die "Virtualenv package install failed."
     }
-    virtualenv "${env_dir}" || {
-        die "Virtualenv creation failed."
+    virtualenv -p $(which python3) "${env_dir}" || {
+        die "Virtualenv creation for $(which python) failed."
     }
     set +u
     source "${env_dir}/bin/activate" || die "Virtualenv activation failed."
     set -u
-    pip install --upgrade -r "${req_path}" || {
+    pip3 install --upgrade -r "${req_path}" || {
         die "Requirements installation failed."
     }
     # Most CSIT Python scripts assume PYTHONPATH is set and exported.
@@ -637,7 +637,8 @@ function run_pybot () {
 
     pushd "${CSIT_DIR}" || die "Change directory operation failed."
     set +e
-    pybot "${all_options[@]}" "${GENERATED_DIR}/tests/"
+    python -V
+    robot "${all_options[@]}" "${GENERATED_DIR}/tests/"
     PYBOT_EXIT_STATUS="$?"
     set -e
 
diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py
index 760d958c54..3f74057832 100644
--- a/resources/libraries/python/SetupFramework.py
+++ b/resources/libraries/python/SetupFramework.py
@@ -111,8 +111,9 @@ def create_env_directory_at_node(node):
                    .format(host))
     exec_cmd_no_error(
         node, 'cd {0} && rm -rf env'
-        ' && virtualenv --system-site-packages --never-download env'
-        ' && source env/bin/activate && pip install -r requirements.txt'
+        ' && virtualenv -p $(which python3) '
+        '--system-site-packages --never-download env'
+        ' && source env/bin/activate && pip3 install -r requirements.txt'
         .format(con.REMOTE_FW_DIR), timeout=100, include_reason=True,
         message="Failed install at node {host}".format(host=host))
     logger.console('Virtualenv setup on {0} done.'.format(host))
diff --git a/resources/tools/doc_gen/run_doc.sh b/resources/tools/doc_gen/run_doc.sh
index 523b4ae505..36535df031 100755
--- a/resources/tools/doc_gen/run_doc.sh
+++ b/resources/tools/doc_gen/run_doc.sh
@@ -33,8 +33,6 @@ virtualenv ${WORKING_DIR}/env
 
 # Install CSIT requirements:
 pip install -r ../../../requirements.txt
-# Install Sphinx:
-pip install -r ${WORKING_DIR}/requirements.txt
 
 export PYTHONPATH=`pwd`
 
diff --git a/resources/tools/doc_gen/src/requirements.txt b/resources/tools/doc_gen/src/requirements.txt
deleted file mode 100644
index 2cf7cd9687..0000000000
--- a/resources/tools/doc_gen/src/requirements.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-Sphinx==1.5.5
-sphinxcontrib_robotdoc
-sphinx-rtd-theme
\ No newline at end of file
diff --git a/resources/tools/presentation/run_cpta.sh b/resources/tools/presentation/run_cpta.sh
index 954f17d53c..a60c1cc7a9 100755
--- a/resources/tools/presentation/run_cpta.sh
+++ b/resources/tools/presentation/run_cpta.sh
@@ -16,11 +16,11 @@ sudo apt-get -y install libxml2 libxml2-dev libxslt-dev build-essential \
 mkdir ${DIR[WORKING]}
 
 # Create virtual environment
-virtualenv ${DIR[WORKING]}/env
-. ${DIR[WORKING]}/env/bin/activate
+virtualenv -p $(which python3) ${DIR[WORKING]}/env
+source ${DIR[WORKING]}/env/bin/activate
 
 # Install python dependencies:
-pip install -r requirements.txt
+pip3 install -r requirements.txt
 
 export PYTHONPATH=`pwd`
 
diff --git a/resources/tools/presentation/run_report.sh b/resources/tools/presentation/run_report.sh
index 24b6c21a70..56c671a8bb 100755
--- a/resources/tools/presentation/run_report.sh
+++ b/resources/tools/presentation/run_report.sh
@@ -27,11 +27,11 @@ fi
 mkdir ${DIR[WORKING]}
 
 # Create virtual environment
-virtualenv ${DIR[WORKING]}/env
-. ${DIR[WORKING]}/env/bin/activate
+virtualenv -p $(which python3) ${DIR[WORKING]}/env
+source ${DIR[WORKING]}/env/bin/activate
 
 # Install python dependencies:
-pip install -r requirements.txt
+pip3 install -r requirements.txt
 
 export PYTHONPATH=`pwd`
 
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
index f53584a939..f2e76a729d 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
@@ -1,8 +1,8 @@
 ---
 # file: lf_inventory/group_vars/all.yaml
 
-# General variables
-ansible_python_interpreter: '/usr/bin/python2.7'
+# Ansible interpreter (for PIP)
+ansible_python_interpreter: 'python3'
 # provision via cobbler
 provision_enabled: False
 # name_servers_search is used in /etc/hosts file on target machine.
diff --git a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml
index 484e6ddb4b..d02746cb4f 100644
--- a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml
@@ -1,8 +1,8 @@
 ---
 # file: group_vars/all.yaml
 
-# General variables
-ansible_python_interpreter: '/usr/bin/python2.7'
+# Ansible interpreter (for PIP)
+ansible_python_interpreter: 'python3'
 # provision via cobbler
 provision_enabled: True
 # name_servers_search is used in /etc/hosts file on target machine.
diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template
index cf2fbdfe34..965562ae0a 100644
--- a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template
+++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template
@@ -21,7 +21,7 @@ option pxe-system-type code 93 = unsigned integer 16;
 subnet 10.30.51.0 netmask 255.255.255.0 {
      option routers             10.30.51.1;
      option domain-name         "linuxfoundation.org";
-     option domain-name-servers 199.204.44.24, 199.204.47.54;
+     option domain-name-servers 1.1.1.1, 8.8.8.8;
      option subnet-mask         255.255.255.0;
      range dynamic-bootp        10.30.51.2 10.30.51.254;
      default-lease-time         600;
diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed
index 250701232c..3827562c06 100644
--- a/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed
+++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed
@@ -114,7 +114,7 @@ d-i user-setup/encrypt-home boolean false
 ### Package selection
 tasksel tasksel/first multiselect ubuntu-server
 # Individual additional packages to install
-d-i pkgsel/include string openssh-server python2.7
+d-i pkgsel/include string openssh-server python2.7 python3.6
 # Whether to upgrade packages after debootstrap.
 # Allowed values: none, safe-upgrade, full-upgrade
 #d-i pkgsel/upgrade select none
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index 3e7e5a00fa..84358bf863 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -70,25 +70,57 @@
 - name: Install CSIT PIP requirements
   pip:
     name:
-      - 'docopt==0.6.2'
-      - 'ecdsa==0.13'
-      - 'enum34==1.1.2'
-      - 'ipaddress==1.0.16'
-      - 'paramiko==1.16.0'
-      - 'pexpect==4.6.0'
+      - 'ecdsa==0.13.3'
+      - 'paramiko==2.6.0'
       - 'pycrypto==2.6.1'
-      - 'pykwalify==1.5.0'
-      - 'pypcap==1.1.5'
-      - 'python-dateutil==2.4.2'
-      - 'PyYAML==3.11'
-      - 'requests==2.9.1'
-      - 'robotframework==2.9.2'
-      - 'scapy==2.3.1'
-      - 'scp==0.10.2'
-      - 'six==1.12.0'
+      - 'pypcap==1.2.3'
+      - 'PyYAML==5.1.1'
+      - 'requests==2.22.0'
+      - 'robotframework==3.1.2'
+      - 'scapy==2.4.3'
+      - 'scp==0.13.2'
+      - 'ansible==2.7.8'
       - 'dill==0.2.8.2'
-      - 'numpy==1.14.5'
-      - 'aenum==2.1.2'
+      - 'numpy==1.17.3'
+      - 'hdrhistogram==0.6.1'
+      - 'pandas==0.25.3'
+      - 'plotly==4.1.1'
+      - 'PTable==0.9.2'
+      - 'Sphinx==2.2.1'
+      - 'sphinx-rtd-theme==0.4.0'
+      - 'sphinxcontrib-programoutput==0.15'
+      - 'sphinxcontrib-robotdoc==0.11.0'
+      - 'alabaster==0.7.12'
+      - 'Babel==2.7.0'
+      - 'bcrypt==3.1.7'
+      - 'certifi==2019.9.11'
+      - 'cffi==1.13.2'
+      - 'chardet==3.0.4'
+      - 'cryptography==2.8'
+      - 'docutils==0.15.2'
+      - 'future==0.18.2'
+      - 'idna==2.8'
+      - 'imagesize==1.1.0'
+      - 'Jinja2==2.10.3'
+      - 'MarkupSafe==1.1.1'
+      - 'packaging==19.2'
+      - 'pbr==5.4.3'
+      - 'pycparser==2.19'
+      - 'Pygments==2.4.2'
+      - 'PyNaCl==1.3.0'
+      - 'pyparsing==2.4.4'
+      - 'python-dateutil==2.8.1'
+      - 'pytz==2019.3'
+      - 'retrying==1.3.3'
+      - 'six==1.13.0'
+      - 'snowballstemmer==2.0.0'
+      - 'sphinxcontrib-applehelp==1.0.1'
+      - 'sphinxcontrib-devhelp==1.0.1'
+      - 'sphinxcontrib-htmlhelp==1.0.2'
+      - 'sphinxcontrib-jsmath==1.0.1'
+      - 'sphinxcontrib-qthelp==1.0.2'
+      - 'sphinxcontrib-serializinghtml==1.1.3'
+      - 'urllib3==1.25.6'
   tags: install-pip
 
 - name: Install CSIT PIP requirements - SciPy workaround