aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--pylint.cfg9
-rw-r--r--requirements.txt118
-rw-r--r--resources/libraries/bash/function/common.sh11
-rw-r--r--resources/libraries/python/SetupFramework.py5
-rwxr-xr-xresources/tools/doc_gen/run_doc.sh2
-rw-r--r--resources/tools/doc_gen/src/requirements.txt3
-rwxr-xr-xresources/tools/presentation/run_cpta.sh6
-rwxr-xr-xresources/tools/presentation/run_report.sh6
-rw-r--r--resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml4
-rw-r--r--resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml4
-rw-r--r--resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template2
-rw-r--r--resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed2
-rw-r--r--resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml66
-rw-r--r--tox.ini4
14 files changed, 150 insertions, 92 deletions
diff --git a/pylint.cfg b/pylint.cfg
index 69668ed79c..5ae4ed551b 100644
--- a/pylint.cfg
+++ b/pylint.cfg
@@ -12,7 +12,7 @@ profile=no
# Add files or directories to the blacklist. They should be base names, not
# paths.
-ignore=CVS
+ignore=CVS, conf.py
# Pickle collected data for later comparisons.
persistent=yes
@@ -282,3 +282,10 @@ max-public-methods=60
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=fstr \ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 3fa2a9d879..24263c7aae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,57 +1,79 @@
# DISCLAIMER: BEFORE EDITING THIS FILE!
#
-# This file has two closely related consequences.
-# The common part is that this file defines
-# the content of virtual Python environment
-# used when Robot tests are running.
-# The difference is in which machine the environment is created on.
-# For the Jenkins executor machines,
-# editing this file has no surprising consequences.
+# This file has two closely related consequences. The common part is that this
+# file defines the content of virtual Python environment used when Robot tests
+# are running.
#
-# But the environment is also being created
-# on virtual machines created by VIRL, which do have specific restrictions.
-# Complete package and setting snapshot (for both VIRL and Physical Testbed),
-# with network being limited in case of VIRL to local only.
-# This implies pip currently being set to site-packages only
-# for both VIRL and Physical Testbed.
-# So if the "virl image" applied on the machine
-# does not reflect the edits, some installations might fail.
-# Even if the failure may not directly impact execution,
-# this leads into inconsistency between installed environment
-# and the code base which may lead into fatal error or uncaught exceptions.
+# But the virtual environment is also being created on containerized
+# environment created by either Nomad cluster (Jenkins, vpp_device) or during
+# execution on SUT machines, which might have specific restrictions.
#
-# The "virl image" is built from files such as
-# resources/tools/disk-image-builder/ubuntu/lists/ubuntu-16.04.1_2017-10-21_2.0/pip-requirements.txt
-# but the build is not automated,
-# the new "virl image" has to be built and applied manually
-# before your edit can pass.
-# This also means, incompatible virl images
-# would make existing tests fail until your edit is merged.
+# Complete package and setting snapshot (for all container images),
+# with network being limited to local only. This implies pip currently being
+# set to site-packages only. So if the "container image" applied on the machine
+# does not reflect the edits, some installations might fail.
+# Even if the failure may not directly impact execution, this leads into
+# inconsistency between installed environment and the code base which may lead
+# into fatal error or uncaught exceptions.
#
-# TODO: Automate the virtualenv download and distribution
-# on all types of testbeds prior the test execution.
-# TODO: Figure out a way to verify edits to this file automatically.
-
-# TODO: Split into directly needed packages and their dependencies.
-aenum==2.1.2
-docopt==0.6.2 # for pykwalify
-ecdsa==0.13
-enum34==1.1.2
-ipaddress==1.0.16
-paramiko==1.16.0
-pexpect==4.6.0
+# CSIT Core dependencies
+ecdsa==0.13.3
+paramiko==2.6.0
pycrypto==2.6.1
-pykwalify==1.5.0
-pypcap==1.1.5
-python-dateutil==2.4.2 # for pykwalify
-PyYAML==3.11
-requests==2.9.1
-robotframework==2.9.2
-scapy==2.3.1
-scp==0.10.2
-six==1.12.0 # for python-dateutil
+pypcap==1.2.3
+PyYAML==5.1.1
+requests==2.22.0
+robotframework==3.1.2
+scapy==2.4.3
+scp==0.13.2
-# The following is only there for PLRsearch.
+# Bootstraping dependencies
+ansible==2.7.8
+
+# PLRsearch dependencies
dill==0.2.8.2
-numpy==1.14.5
+numpy==1.17.3
scipy==1.1.0
+
+# PAL/DOC dependencies
+hdrhistogram==0.6.1
+pandas==0.25.3
+plotly==4.1.1
+PTable==0.9.2
+Sphinx==2.2.1
+sphinx-rtd-theme==0.4.0
+sphinxcontrib-programoutput==0.15
+sphinxcontrib-robotdoc==0.11.0
+
+# PIP freeze dependencies
+alabaster==0.7.12
+Babel==2.7.0
+bcrypt==3.1.7
+certifi==2019.9.11
+cffi==1.13.2
+chardet==3.0.4
+cryptography==2.8
+docutils==0.15.2
+future==0.18.2
+idna==2.8
+imagesize==1.1.0
+Jinja2==2.10.3
+MarkupSafe==1.1.1
+packaging==19.2
+pbr==5.4.3
+pycparser==2.19
+Pygments==2.4.2
+PyNaCl==1.3.0
+pyparsing==2.4.4
+python-dateutil==2.8.1
+pytz==2019.3
+retrying==1.3.3
+six==1.13.0
+snowballstemmer==2.0.0
+sphinxcontrib-applehelp==1.0.1
+sphinxcontrib-devhelp==1.0.1
+sphinxcontrib-htmlhelp==1.0.2
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.2
+sphinxcontrib-serializinghtml==1.1.3
+urllib3==1.25.6
diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index 1ef7c93da3..3a7a0fbf7d 100644
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -113,16 +113,16 @@ function activate_virtualenv () {
env_dir="${root_path}/env"
req_path=${2-$CSIT_DIR/requirements.txt}
rm -rf "${env_dir}" || die "Failed to clean previous virtualenv."
- pip install --upgrade virtualenv || {
+ pip3 install --upgrade virtualenv || {
die "Virtualenv package install failed."
}
- virtualenv "${env_dir}" || {
- die "Virtualenv creation failed."
+ virtualenv -p $(which python3) "${env_dir}" || {
+ die "Virtualenv creation for $(which python) failed."
}
set +u
source "${env_dir}/bin/activate" || die "Virtualenv activation failed."
set -u
- pip install --upgrade -r "${req_path}" || {
+ pip3 install --upgrade -r "${req_path}" || {
die "Requirements installation failed."
}
# Most CSIT Python scripts assume PYTHONPATH is set and exported.
@@ -637,7 +637,8 @@ function run_pybot () {
pushd "${CSIT_DIR}" || die "Change directory operation failed."
set +e
- pybot "${all_options[@]}" "${GENERATED_DIR}/tests/"
+ python -V
+ robot "${all_options[@]}" "${GENERATED_DIR}/tests/"
PYBOT_EXIT_STATUS="$?"
set -e
diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py
index 760d958c54..3f74057832 100644
--- a/resources/libraries/python/SetupFramework.py
+++ b/resources/libraries/python/SetupFramework.py
@@ -111,8 +111,9 @@ def create_env_directory_at_node(node):
.format(host))
exec_cmd_no_error(
node, 'cd {0} && rm -rf env'
- ' && virtualenv --system-site-packages --never-download env'
- ' && source env/bin/activate && pip install -r requirements.txt'
+ ' && virtualenv -p $(which python3) '
+ '--system-site-packages --never-download env'
+ ' && source env/bin/activate && pip3 install -r requirements.txt'
.format(con.REMOTE_FW_DIR), timeout=100, include_reason=True,
message="Failed install at node {host}".format(host=host))
logger.console('Virtualenv setup on {0} done.'.format(host))
diff --git a/resources/tools/doc_gen/run_doc.sh b/resources/tools/doc_gen/run_doc.sh
index 523b4ae505..36535df031 100755
--- a/resources/tools/doc_gen/run_doc.sh
+++ b/resources/tools/doc_gen/run_doc.sh
@@ -33,8 +33,6 @@ virtualenv ${WORKING_DIR}/env
# Install CSIT requirements:
pip install -r ../../../requirements.txt
-# Install Sphinx:
-pip install -r ${WORKING_DIR}/requirements.txt
export PYTHONPATH=`pwd`
diff --git a/resources/tools/doc_gen/src/requirements.txt b/resources/tools/doc_gen/src/requirements.txt
deleted file mode 100644
index 2cf7cd9687..0000000000
--- a/resources/tools/doc_gen/src/requirements.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-Sphinx==1.5.5
-sphinxcontrib_robotdoc
-sphinx-rtd-theme \ No newline at end of file
diff --git a/resources/tools/presentation/run_cpta.sh b/resources/tools/presentation/run_cpta.sh
index 954f17d53c..a60c1cc7a9 100755
--- a/resources/tools/presentation/run_cpta.sh
+++ b/resources/tools/presentation/run_cpta.sh
@@ -16,11 +16,11 @@ sudo apt-get -y install libxml2 libxml2-dev libxslt-dev build-essential \
mkdir ${DIR[WORKING]}
# Create virtual environment
-virtualenv ${DIR[WORKING]}/env
-. ${DIR[WORKING]}/env/bin/activate
+virtualenv -p $(which python3) ${DIR[WORKING]}/env
+source ${DIR[WORKING]}/env/bin/activate
# Install python dependencies:
-pip install -r requirements.txt
+pip3 install -r requirements.txt
export PYTHONPATH=`pwd`
diff --git a/resources/tools/presentation/run_report.sh b/resources/tools/presentation/run_report.sh
index 24b6c21a70..56c671a8bb 100755
--- a/resources/tools/presentation/run_report.sh
+++ b/resources/tools/presentation/run_report.sh
@@ -27,11 +27,11 @@ fi
mkdir ${DIR[WORKING]}
# Create virtual environment
-virtualenv ${DIR[WORKING]}/env
-. ${DIR[WORKING]}/env/bin/activate
+virtualenv -p $(which python3) ${DIR[WORKING]}/env
+source ${DIR[WORKING]}/env/bin/activate
# Install python dependencies:
-pip install -r requirements.txt
+pip3 install -r requirements.txt
export PYTHONPATH=`pwd`
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
index f53584a939..f2e76a729d 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml
@@ -1,8 +1,8 @@
---
# file: lf_inventory/group_vars/all.yaml
-# General variables
-ansible_python_interpreter: '/usr/bin/python2.7'
+# Ansible interpreter (for PIP)
+ansible_python_interpreter: 'python3'
# provision via cobbler
provision_enabled: False
# name_servers_search is used in /etc/hosts file on target machine.
diff --git a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml
index 484e6ddb4b..d02746cb4f 100644
--- a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml
+++ b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml
@@ -1,8 +1,8 @@
---
# file: group_vars/all.yaml
-# General variables
-ansible_python_interpreter: '/usr/bin/python2.7'
+# Ansible interpreter (for PIP)
+ansible_python_interpreter: 'python3'
# provision via cobbler
provision_enabled: True
# name_servers_search is used in /etc/hosts file on target machine.
diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template
index cf2fbdfe34..965562ae0a 100644
--- a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template
+++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template
@@ -21,7 +21,7 @@ option pxe-system-type code 93 = unsigned integer 16;
subnet 10.30.51.0 netmask 255.255.255.0 {
option routers 10.30.51.1;
option domain-name "linuxfoundation.org";
- option domain-name-servers 199.204.44.24, 199.204.47.54;
+ option domain-name-servers 1.1.1.1, 8.8.8.8;
option subnet-mask 255.255.255.0;
range dynamic-bootp 10.30.51.2 10.30.51.254;
default-lease-time 600;
diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed
index 250701232c..3827562c06 100644
--- a/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed
+++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed
@@ -114,7 +114,7 @@ d-i user-setup/encrypt-home boolean false
### Package selection
tasksel tasksel/first multiselect ubuntu-server
# Individual additional packages to install
-d-i pkgsel/include string openssh-server python2.7
+d-i pkgsel/include string openssh-server python2.7 python3.6
# Whether to upgrade packages after debootstrap.
# Allowed values: none, safe-upgrade, full-upgrade
#d-i pkgsel/upgrade select none
diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
index 3e7e5a00fa..84358bf863 100644
--- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml
@@ -70,25 +70,57 @@
- name: Install CSIT PIP requirements
pip:
name:
- - 'docopt==0.6.2'
- - 'ecdsa==0.13'
- - 'enum34==1.1.2'
- - 'ipaddress==1.0.16'
- - 'paramiko==1.16.0'
- - 'pexpect==4.6.0'
+ - 'ecdsa==0.13.3'
+ - 'paramiko==2.6.0'
- 'pycrypto==2.6.1'
- - 'pykwalify==1.5.0'
- - 'pypcap==1.1.5'
- - 'python-dateutil==2.4.2'
- - 'PyYAML==3.11'
- - 'requests==2.9.1'
- - 'robotframework==2.9.2'
- - 'scapy==2.3.1'
- - 'scp==0.10.2'
- - 'six==1.12.0'
+ - 'pypcap==1.2.3'
+ - 'PyYAML==5.1.1'
+ - 'requests==2.22.0'
+ - 'robotframework==3.1.2'
+ - 'scapy==2.4.3'
+ - 'scp==0.13.2'
+ - 'ansible==2.7.8'
- 'dill==0.2.8.2'
- - 'numpy==1.14.5'
- - 'aenum==2.1.2'
+ - 'numpy==1.17.3'
+ - 'hdrhistogram==0.6.1'
+ - 'pandas==0.25.3'
+ - 'plotly==4.1.1'
+ - 'PTable==0.9.2'
+ - 'Sphinx==2.2.1'
+ - 'sphinx-rtd-theme==0.4.0'
+ - 'sphinxcontrib-programoutput==0.15'
+ - 'sphinxcontrib-robotdoc==0.11.0'
+ - 'alabaster==0.7.12'
+ - 'Babel==2.7.0'
+ - 'bcrypt==3.1.7'
+ - 'certifi==2019.9.11'
+ - 'cffi==1.13.2'
+ - 'chardet==3.0.4'
+ - 'cryptography==2.8'
+ - 'docutils==0.15.2'
+ - 'future==0.18.2'
+ - 'idna==2.8'
+ - 'imagesize==1.1.0'
+ - 'Jinja2==2.10.3'
+ - 'MarkupSafe==1.1.1'
+ - 'packaging==19.2'
+ - 'pbr==5.4.3'
+ - 'pycparser==2.19'
+ - 'Pygments==2.4.2'
+ - 'PyNaCl==1.3.0'
+ - 'pyparsing==2.4.4'
+ - 'python-dateutil==2.8.1'
+ - 'pytz==2019.3'
+ - 'retrying==1.3.3'
+ - 'six==1.13.0'
+ - 'snowballstemmer==2.0.0'
+ - 'sphinxcontrib-applehelp==1.0.1'
+ - 'sphinxcontrib-devhelp==1.0.1'
+ - 'sphinxcontrib-htmlhelp==1.0.2'
+ - 'sphinxcontrib-jsmath==1.0.1'
+ - 'sphinxcontrib-qthelp==1.0.2'
+ - 'sphinxcontrib-serializinghtml==1.1.3'
+ - 'urllib3==1.25.6'
tags: install-pip
- name: Install CSIT PIP requirements - SciPy workaround
diff --git a/tox.ini b/tox.ini
index b235bf3ab8..4617e73e54 100644
--- a/tox.ini
+++ b/tox.ini
@@ -36,9 +36,9 @@ checker_dir = ./resources/libraries/bash/entry/check
# TODO: Tox prints various warnings. Figure them out and fix them.
[testenv:pylint]
-basepython = python2
+basepython = python3
deps =
- pylint==1.5.4
+ pylint==2.4.3
-r ./requirements.txt
whitelist_externals = /bin/bash
setenv = PYTHONPATH = {toxinidir}