diff options
Diffstat (limited to 'resources/libraries')
17 files changed, 1014 insertions, 13 deletions
diff --git a/resources/libraries/bash/entry/check/tc_naming.sh b/resources/libraries/bash/entry/check/tc_naming.sh index e9f86fc0dd..bc2ac32671 100644 --- a/resources/libraries/bash/entry/check/tc_naming.sh +++ b/resources/libraries/bash/entry/check/tc_naming.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -48,14 +48,14 @@ r_parse='(.*)\/(.*).robot.*(([0-9]{2,4}B|IMIX)-.*)' # One caveat of this solution is that we cannot proceed to check full names now # as majority of Testcases does not meet naming criteria. s_testc_rules=( - 'packet size' + 'packet size or file size' 'core combination' 'NIC driver mode' 'packet encapsulation on L2 layer' 'test type' ) r_testc_rules=( - '^([[:digit:]]{2,4}B|IMIX)-' + '^([[:digit:]]{1,4}B|IMIX)-' '([[:digit:]]+c-){0,1}' '(avf-|1lbvpplacp-|2lbvpplacp-){0,1}' '(eth|dot1q|dot1ad)' diff --git a/resources/libraries/bash/entry/install_nginx.sh b/resources/libraries/bash/entry/install_nginx.sh new file mode 100755 index 0000000000..3a2c8ef7eb --- /dev/null +++ b/resources/libraries/bash/entry/install_nginx.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021 Intel and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install the NGINX framework on the DUT node. Check prerequisites. + +set -exuo pipefail + +# Assumptions: +# + There is a directory holding CSIT code to use (this script is there). +# + At least one of the following is true: +# ++ JOB_NAME environment variable is set, +# ++ or this entry script has access to arguments. +# Consequences (and specific assumptions) are multiple, +# examine tree of functions for current description. + +BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")" +source "${BASH_FUNCTION_DIR}/common.sh" || { + echo "Source failed." >&2 + exit 1 +} +source "${BASH_FUNCTION_DIR}/nginx.sh" || die "Source failed." +common_dirs ${@} || die +gather_nginx || die "Download nginx failed." +nginx_extract || die "Extract nginx failed." +nginx_compile || die "Compile nginx failed." diff --git a/resources/libraries/bash/function/nginx.sh b/resources/libraries/bash/function/nginx.sh new file mode 100755 index 0000000000..122af23852 --- /dev/null +++ b/resources/libraries/bash/function/nginx.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021 Intel and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + + +function gather_nginx () { + + # Ensure stable NGINX archive is downloaded. + # + # Variables read: + # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from. + # - NGINX_VER - Version number of Nginx. + set -exuo pipefail + pushd "${DOWNLOAD_DIR}" || die "Pushd failed." + nginx_repo="http://nginx.org/download/" + # Use downloaded packages with specific version + echo "Downloading NGINX package of specific version from repo ..." + # Downloading NGINX version based on what VPP is using. Currently + # it is not easy way to detect from VPP version automatically. + nginx_stable_ver="${NGINX_VER}".tar.gz + + if [[ ! -f "${nginx_stable_ver}" ]]; then + wget -nv --no-check-certificate \ + "${nginx_repo}/${nginx_stable_ver}" || { + die "Failed to get NGINX package from: ${nginx_repo}." + } + fi + popd || die "Popd failed." +} + + +function common_dirs () { + + # Set global variables, create some directories (without touching content). + # This function assumes running in remote testbed. It might override other + # functions if included from common.sh. + + # Arguments: + # - ${1} - Version number of Nginx. + # Variables set: + # - BASH_FUNCTION_DIR - Path to existing directory this file is located in. + # - CSIT_DIR - Path to CSIT framework. + # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from. + # - NGINX_DIR - Path to NGINX framework. + # - NGINX_VER - Version number of Nginx. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + NGINX_VER="${1}" + this_file=$(readlink -e "${BASH_SOURCE[0]}") || { + die "Some error during locating of this source file." + } + BASH_FUNCTION_DIR=$(dirname "${this_file}") || { + die "Some error during dirname call." + } + CSIT_DIR=$(readlink -e "/tmp/openvpp-testing") || { + die "Readlink failed." + } + DOWNLOAD_DIR=$(readlink -f "${CSIT_DIR}/download_dir") || { + die "Readlink failed." + } + mkdir -p "${CSIT_DIR}/${NGINX_VER}" || die "Mkdir failed." + NGINX_DIR=$(readlink -e "${CSIT_DIR}/${NGINX_VER}") || { + die "Readlink failed." + } +} + + + +function nginx_compile () { + + # Compile NGINX archive. + # + # Variables read: + # - NGINX_DIR - Path to NGINX framework. + # - CSIT_DIR - Path to CSIT framework. + # - NGINX_INS_PATH - Path to NGINX install path. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + NGINX_INS_PATH="${DOWNLOAD_DIR}/${NGINX_VER}" + pushd "${NGINX_DIR}" || die "Pushd failed." + + # Set installation prefix. + param="--prefix=${NGINX_INS_PATH} " + # Set nginx binary pathname. + param+="--sbin-path=${NGINX_INS_PATH}/sbin/nginx " + # Set nginx.conf pathname. + param+="--conf-path=${NGINX_INS_PATH}/conf/nginx.conf " + # Enable ngx_http_stub_status_module. + param+="--with-http_stub_status_module " + # Force PCRE library usage. + param+="--with-pcre " + # Enable ngx_http_realip_module. + param+="--with-http_realip_module " + params=(${param}) + ./configure "${params[@]}" || die "Failed to configure NGINX!" + make -j 16;make install || die "Failed to compile NGINX!" +} + + +function nginx_extract () { + + # Extract NGINX framework. + # + # Variables read: + # - NGINX_DIR - Path to NGINX framework. + # - CSIT_DIR - Path to CSIT framework. + # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from. + # - NGINX_VER - Version number of Nginx. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + + pushd "${CSIT_DIR}" || die "Pushd failed." + tar -xvf ${DOWNLOAD_DIR}/${NGINX_VER}.tar.gz --strip=1 \ + --directory "${NGINX_DIR}" || { + die "Failed to extract NGINX!" + } +} diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py index 79b94be7ff..be9fe34915 100644 --- a/resources/libraries/python/Constants.py +++ b/resources/libraries/python/Constants.py @@ -130,6 +130,9 @@ class Constants: # shell scripts location RESOURCES_LIB_SH = u"resources/libraries/bash" + # python scripts location + RESOURCES_LIB_PY = u"resources/libraries/python" + # Python API provider location RESOURCES_PAPI_PROVIDER = u"resources/tools/papi/vpp_papi_provider.py" diff --git a/resources/libraries/python/CpuUtils.py b/resources/libraries/python/CpuUtils.py index 170cbe6b2e..293d6b6913 100644 --- a/resources/libraries/python/CpuUtils.py +++ b/resources/libraries/python/CpuUtils.py @@ -255,7 +255,7 @@ class CpuUtils: cpu_list_0 = cpu_list[:cpu_list_len // CpuUtils.NR_OF_THREADS] cpu_list_1 = cpu_list[cpu_list_len // CpuUtils.NR_OF_THREADS:] cpu_range = f"{cpu_list_0[0]}{sep}{cpu_list_0[-1]}," \ - f"{cpu_list_1[0]}{sep}{cpu_list_1[-1]}" + f"{cpu_list_1[0]}{sep}{cpu_list_1[-1]}" else: cpu_range = f"{cpu_list[0]}{sep}{cpu_list[-1]}" @@ -469,3 +469,25 @@ class CpuUtils: return CpuUtils.cpu_slice_of_list_per_node( node, cpu_node=cpu_node, skip_cnt=skip_cnt, cpu_cnt=cpu_cnt, smt_used=False) + + @staticmethod + def get_cpu_idle_list(node, cpu_node, smt_used, cpu_alloc_str, sep=u","): + """ + Get idle CPU List + :param node: Node dictionary with cpuinfo. + :param cpu_node: Numa node number. + :param smt_used: True - we want to use SMT, otherwise false. + :param cpu_alloc_str: vpp used cores. + :param sep: Separator, default: ",". + :type node: dict + :type cpu_node: int + :type smt_used: bool + :type cpu_alloc_str: str + :type smt_used: bool + :type sep: str + :rtype: list + """ + cpu_list = CpuUtils.cpu_list_per_node(node, cpu_node, smt_used) + cpu_idle_list = [i for i in cpu_list + if str(i) not in cpu_alloc_str.split(sep)] + return cpu_idle_list diff --git a/resources/libraries/python/HoststackUtil.py b/resources/libraries/python/HoststackUtil.py index c307946698..e797c3c206 100644 --- a/resources/libraries/python/HoststackUtil.py +++ b/resources/libraries/python/HoststackUtil.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -165,6 +165,39 @@ class HoststackUtil(): return stdout_log, stderr_log @staticmethod + def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir): + """Construct the NGINX command using the specified attributes. + + :param nginx_attributes: NGINX test program attributes. + :param nginx_version: NGINX version. + :param nginx_ins_dir: NGINX install dir. + :type nginx_attributes: dict + :type nginx_version: str + :type nginx_ins_dir: str + :returns: Command line components of the NGINX command + 'env_vars' - environment variables + 'name' - program name + 'args' - command arguments. + 'path' - program path. + :rtype: dict + """ + nginx_cmd = dict() + nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \ + f"{Constants.RESOURCES_TPL_VCL}/" \ + f"{nginx_attributes[u'vcl_config']}" + if nginx_attributes[u"ld_preload"]: + nginx_cmd[u"env_vars"] += \ + f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}" + if nginx_attributes[u'transparent_tls']: + nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1" + + nginx_cmd[u"name"] = u"nginx" + nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/" + nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \ + f"nginx-{nginx_version}/conf/nginx.conf" + return nginx_cmd + + @staticmethod def start_hoststack_test_program(node, namespace, core_list, program): """Start the specified HostStack test program. @@ -194,9 +227,13 @@ class HoststackUtil(): env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u"" args = program[u"args"] - cmd = f"nohup {shell_cmd} \'{env_vars}taskset --cpu-list {core_list} " \ - f"{program_name} {args} >/tmp/{program_name}_stdout.log " \ - f"2>/tmp/{program_name}_stderr.log &\'" + program_path = program.get(u"path", u"") + # NGINX used `worker_cpu_affinity` in configuration file + taskset_cmd = u"" if program_name == u"nginx" else \ + f"taskset --cpu-list {core_list}" + cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \ + f"{program_path}{program_name} {args} >/tmp/{program_name}_" \ + f"stdout.log 2>/tmp/{program_name}_stderr.log &\'" try: exec_cmd_no_error(node, cmd, sudo=True) return DUTSetup.get_pid(node, program_name)[0] @@ -350,3 +387,18 @@ class HoststackUtil(): :rtype: bool """ return server_defer_fail and client_defer_fail + + @staticmethod + def log_vpp_hoststack_data(node): + """Retrieve and log VPP HostStack data. + + :param node: DUT node. + :type node: dict + :raises RuntimeError: If node subtype is not a DUT or startup failed. + """ + + if node[u"type"] != u"DUT": + raise RuntimeError(u"Node type is not a DUT!") + + PapiSocketExecutor.run_cli_cmd(node, u"show error") + PapiSocketExecutor.run_cli_cmd(node, u"show interface") diff --git a/resources/libraries/python/NGINX/NGINXTools.py b/resources/libraries/python/NGINX/NGINXTools.py new file mode 100644 index 0000000000..9418484f15 --- /dev/null +++ b/resources/libraries/python/NGINX/NGINXTools.py @@ -0,0 +1,145 @@ +# Copyright (c) 2021 Intel and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""This module implements initialization and cleanup of NGINX framework.""" + +from robot.api import logger + +from resources.libraries.python.Constants import Constants +from resources.libraries.python.ssh import exec_cmd_no_error, exec_cmd +from resources.libraries.python.topology import NodeType +from resources.libraries.python.NginxUtil import NginxUtil + + +class NGINXTools: + """This class implements: + - Initialization of NGINX environment, + - Cleanup of NGINX environment. + """ + + @staticmethod + def cleanup_nginx_framework(node, nginx_ins_path): + """ + Cleanup the NGINX framework on the DUT node. + + :param node: Will cleanup the nginx on this nodes. + :param nginx_ins_path: NGINX install path. + :type node: dict + :type nginx_ins_path: str + :raises RuntimeError: If it fails to cleanup the nginx. + """ + check_path_cmd = NginxUtil.get_cmd_options(path=nginx_ins_path) + exec_cmd_no_error(node, check_path_cmd, timeout=180, + message=u"Check NGINX install path failed!") + command = f"rm -rf {nginx_ins_path}" + message = u"Cleanup the NGINX failed!" + exec_cmd_no_error(node, command, timeout=180, message=message) + + @staticmethod + def cleanup_nginx_framework_on_all_duts(nodes, nginx_ins_path): + """ + Cleanup the NGINX framework on all DUT nodes. + + :param nodes: Will cleanup the nginx on this nodes. + :param nginx_ins_path: NGINX install path. + :type nodes: dict + :type nginx_ins_path: str + :raises RuntimeError: If it fails to cleanup the nginx. + """ + for node in nodes.values(): + if node[u"type"] == NodeType.DUT: + NGINXTools.cleanup_nginx_framework(node, nginx_ins_path) + + @staticmethod + def install_original_nginx_framework(node, pkg_dir, nginx_version): + """ + Prepare the NGINX framework on the DUT node. + + :param node: Node from topology file. + :param pkg_dir: Ldp NGINX install dir. + :param nginx_version: NGINX Version. + :type node: dict + :type pkg_dir: str + :type nginx_version: str + :raises RuntimeError: If command returns nonzero return code. + """ + nginx_path = f"{pkg_dir}/nginx-{nginx_version}/sbin/nginx" + cmd_options = NginxUtil.get_cmd_options(path=nginx_path) + ret_code, _, stderr = exec_cmd(node, cmd_options, sudo=True) + if nginx_version in stderr and ret_code == 0: + logger.info(f"NGINX Version: {stderr}") + return + command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}" \ + f"/entry/install_nginx.sh nginx-{nginx_version}" + message = u"Install the NGINX failed!" + exec_cmd_no_error(node, command, sudo=True, timeout=600, + message=message) + _, stderr = exec_cmd_no_error(node, cmd_options, sudo=True, + message=message) + + logger.info(f"NGINX Version: {stderr}") + + @staticmethod + def install_vsap_nginx_on_dut(node, pkg_dir): + """ + Prepare the VSAP NGINX framework on all DUT + + :param node: Node from topology file. + :param pkg_dir: Path to directory where packages are stored. + :type node: dict + :type pkg_dir: str + :raises RuntimeError: If command returns nonzero return code. + """ + command = u". /etc/lsb-release; echo \"${DISTRIB_ID}\"" + stdout, _ = exec_cmd_no_error(node, command) + + if stdout.strip() == u"Ubuntu": + logger.console(u"NGINX install on DUT... ") + exec_cmd_no_error( + node, u"apt-get purge -y 'vsap*' || true", timeout=120, + sudo=True + ) + exec_cmd_no_error( + node, f"dpkg -i --force-all {pkg_dir}vsap-nginx*.deb", + timeout=120, sudo=True, + message=u"Installation of vsap-nginx failed!" + ) + + exec_cmd_no_error(node, u"dpkg -l | grep vsap*", + sudo=True) + + logger.console(u"Completed!\n") + else: + logger.console(u"Ubuntu need!\n") + + @staticmethod + def install_nginx_framework_on_all_duts(nodes, pkg_dir, nginx_version=None): + """ + Prepare the NGINX framework on all DUTs. + + :param nodes: Nodes from topology file. + :param pkg_dir: Path to directory where packages are stored. + :param nginx_version: NGINX version. + :type nodes: dict + :type pkg_dir: str + :type nginx_version: str + """ + + for node in list(nodes.values()): + if node[u"type"] == NodeType.DUT: + if nginx_version: + NGINXTools.install_original_nginx_framework(node, pkg_dir, + nginx_version) + else: + NGINXTools.install_vsap_nginx_on_dut(node, pkg_dir) diff --git a/resources/libraries/python/NGINX/__init__.py b/resources/libraries/python/NGINX/__init__.py new file mode 100644 index 0000000000..d828cbe7cb --- /dev/null +++ b/resources/libraries/python/NGINX/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2021 Intel and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +__init__ file for directory resources/libraries/python/NGINX +""" diff --git a/resources/libraries/python/NginxConfigGenerator.py b/resources/libraries/python/NginxConfigGenerator.py new file mode 100644 index 0000000000..1a0f5f077a --- /dev/null +++ b/resources/libraries/python/NginxConfigGenerator.py @@ -0,0 +1,244 @@ +# Copyright (c) 2021 Intel and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Nginx Configuration File Generator library. +""" + +from resources.libraries.python.ssh import exec_cmd_no_error +from resources.libraries.python.topology import NodeType +from resources.libraries.python.NginxUtil import NginxUtil + +__all__ = [u"NginxConfigGenerator"] + + +class NginxConfigGenerator: + """NGINX Configuration File Generator.""" + + def __init__(self): + """Initialize library.""" + # VPP Node to apply configuration on + self._node = u"" + # NGINX Startup config location + self._nginx_path = u"/usr/local/nginx/" + # Serialized NGinx Configuration + self._nginx_config = u"" + # VPP Configuration + self._nodeconfig = dict() + + def set_node(self, node): + """Set DUT node. + + :param node: Node to store configuration on. + :type node: dict + :raises RuntimeError: If Node type is not DUT. + """ + if node[u"type"] != NodeType.DUT: + raise RuntimeError( + u"Startup config can only be applied to DUTnode." + ) + self._node = node + + def set_nginx_path(self, packages_dir, nginx_version): + """Set NGINX Conf Name. + + :param packages_dir: NGINX install path. + :param nginx_version: Test NGINX version. + :type packages_dir: str + :type nginx_version: str + :raises RuntimeError: If Node type is not DUT. + """ + if nginx_version: + self._nginx_path = f"{packages_dir}/nginx-{nginx_version}" + + def add_http_server_listen(self, value): + """Add Http Server listen port configuration.""" + path = [u"http", u"server", u"listen"] + self.add_config_item(self._nodeconfig, value, path) + + def add_http_server_root(self, value=u"html"): + """Add Http Server root configuration.""" + path = [u"http", u"server", u"root"] + self.add_config_item(self._nodeconfig, value, path) + + def add_http_server_index(self, value=u"index.html index.htm"): + """Add Http Server index configuration.""" + path = [u"http", u"server", u"index"] + self.add_config_item(self._nodeconfig, value, path) + + def add_config_item(self, config, value, path): + """Add NGINX configuration item. + + :param config: Startup configuration of node. + :param value: Value to insert. + :param path: Path where to insert item. + :type config: dict + :type value: str + :type path: list + """ + if len(path) == 1: + config[path[0]] = value + return + if path[0] not in config: + config[path[0]] = dict() + elif isinstance(config[path[0]], str): + config[path[0]] = dict() if config[path[0]] == u"" \ + else {config[path[0]]: u""} + self.add_config_item(config[path[0]], value, path[1:]) + + def dump_config(self, obj, level=-1): + """Dump the startup configuration in NGINX config format. + + :param obj: Python Object to print. + :param level: Nested level for indentation. + :type obj: Obj + :type level: int + :returns: nothing + """ + indent = u" " + if level >= 0: + self._nginx_config += f"{level * indent}{{\n" + if isinstance(obj, dict): + for key, val in obj.items(): + if hasattr(val, u"__iter__") and not isinstance(val, str): + self._nginx_config += f"{(level + 1) * indent}{key}\n" + self.dump_config(val, level + 1) + else: + self._nginx_config += f"{(level + 1) * indent}" \ + f"{key} {val};\n" + else: + for val in obj: + self._nginx_config += f"{(level + 1) * indent}{val};\n" + if level >= 0: + self._nginx_config += f"{level * indent}}}\n" + + def write_config(self, filename=None): + """Generate and write NGINX startup configuration to file. + + :param filename: NGINX configuration file name. + :type filename: str + """ + if filename is None: + filename = f"{self._nginx_path}/conf/nginx.conf" + self.dump_config(self._nodeconfig) + cmd = f"echo \"{self._nginx_config}\" | sudo tee {filename}" + exec_cmd_no_error( + self._node, cmd, message=u"Writing config file failed!" + ) + + def add_http_server_location(self, size): + """Add Http Server location configuration. + + :param size: File size. + :type size: int + """ + if size == 0: + files = u"return" + elif size >= 1024: + files = f"{int(size / 1024)}KB.json" + else: + files = f"{size}B.json" + key = f"{files}" + size_str = size * u"x" + value = "200 '%s'" % size_str + path = [u"http", u"server", f"location /{key}", u"return"] + self.add_config_item(self._nodeconfig, value, path) + + def add_http_access_log(self, value=u"off"): + """Add Http access_log configuration.""" + path = [u"http", u"access_log"] + self.add_config_item(self._nodeconfig, value, path) + + def add_http_include(self, value=u"mime.types"): + """Add Http include configuration.""" + path = [u"http", u"include"] + self.add_config_item(self._nodeconfig, value, path) + + def add_http_default_type(self, value=u"application/octet-stream"): + """Add Http default_type configuration.""" + path = [u"http", u"default_type"] + self.add_config_item(self._nodeconfig, value, path) + + def add_http_sendfile(self, value=u"on"): + """Add Http sendfile configuration.""" + path = [u"http", u"sendfile"] + self.add_config_item(self._nodeconfig, value, path) + + def add_http_keepalive_timeout(self, value): + """Add Http keepalive alive timeout configuration.""" + path = [u"http", u"keepalive_timeout"] + self.add_config_item(self._nodeconfig, value, path) + + def add_http_keepalive_requests(self, value): + """Add Http keepalive alive requests configuration.""" + path = [u"http", u"keepalive_requests"] + self.add_config_item(self._nodeconfig, value, path) + + def add_events_use(self, value=u"epoll"): + """Add Events use configuration.""" + path = [u"events", u"use"] + self.add_config_item(self._nodeconfig, value, path) + + def add_events_worker_connections(self, value=10240): + """Add Events worker connections configuration.""" + path = [u"events", u"worker_connections"] + self.add_config_item(self._nodeconfig, value, path) + + def add_events_accept_mutex(self, value=u"off"): + """Add Events accept mutex configuration.""" + path = [u"events", u"accept_mutex"] + self.add_config_item(self._nodeconfig, value, path) + + def add_events_multi_accept(self, value=u"off"): + """Add Events multi accept configuration.""" + path = [u"events", u"multi_accept"] + self.add_config_item(self._nodeconfig, value, path) + + def add_worker_rlimit_nofile(self, value=10240): + """Add Events worker rlimit nofile configuration.""" + path = [u"worker_rlimit_nofile"] + self.add_config_item(self._nodeconfig, value, path) + + def add_master_process(self, value=u"on"): + """Add master process configuration.""" + path = [u"master_process"] + self.add_config_item(self._nodeconfig, value, path) + + def add_daemon(self, value=u"off"): + """Add daemon configuration.""" + path = [u"daemon"] + self.add_config_item(self._nodeconfig, value, path) + + def add_worker_processes(self, value, smt_used): + """Add worker processes configuration.""" + # nginx workers : vpp used phy workers = 2:1 + if smt_used: + value = value * 4 + else: + value = value * 2 + path = [u"worker_processes"] + self.add_config_item(self._nodeconfig, value, path) + + def apply_config(self, filename=None, verify_nginx=True): + """Generate and write NGINX configuration to file and + verify configuration. + + :param filename: NGINX configuration file name. + :param verify_nginx: Verify NGINX configuration. + :type filename: str + :type verify_nginx: bool + """ + self.write_config(filename=filename) + + app_path = f"{self._nginx_path}/sbin/nginx" + if verify_nginx: + NginxUtil.nginx_config_verify(self._node, app_path) diff --git a/resources/libraries/python/NginxUtil.py b/resources/libraries/python/NginxUtil.py new file mode 100644 index 0000000000..a19ac37291 --- /dev/null +++ b/resources/libraries/python/NginxUtil.py @@ -0,0 +1,124 @@ +# Copyright (c) 2021 Intel and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""NGINX Utilities Library.""" + +from resources.libraries.python.OptionString import OptionString +from resources.libraries.python.ssh import exec_cmd_no_error +from resources.libraries.python.topology import NodeType +from resources.libraries.python.DUTSetup import DUTSetup + + +class NginxUtil: + """Utilities for NGINX.""" + + @staticmethod + def get_cmd_options(**kwargs): + """Create parameters options. + + :param kwargs: Dict of cmd parameters. + :type kwargs: dict + :returns: cmd parameters. + :rtype: OptionString + """ + cmd_options = OptionString() + nginx_path = kwargs.get(u"path", u"/usr/local/nginx") + cmd_options.add(nginx_path) + options = OptionString(prefix=u"-") + # Show Nginx Version + options.add(u"v") + # Verify Configuration + options.add(u"t") + # Send signal to a master process: stop, quit, reopen. + options.add_with_value_from_dict( + u"s", u"signal", kwargs + ) + # Set prefix path (default: /usr/local/nginx/). + options.add_with_value_from_dict( + u"p", u"prefix", kwargs + ) + # Set configuration file (default: conf/nginx.conf). + options.add_with_value_from_dict( + u"c", u"filename", kwargs + ) + # Set global directives out of configuration file + options.add_with_value_from_dict( + u"g", u"directives", kwargs + ) + cmd_options.extend(options) + return cmd_options + + @staticmethod + def nginx_cmd_stop(node, path): + """Stop NGINX cmd app on node. + :param node: Topology node. + :param path: Nginx install path. + :type node: dict + :type path: str + :returns: nothing + """ + cmd_options = NginxUtil.get_cmd_options(path=path, signal=u"stop") + + exec_cmd_no_error(node, cmd_options, sudo=True, disconnect=True, + message=u"Nginx stop failed!") + + @staticmethod + def nginx_cmd_start(node, path, filename): + """Start NGINX cmd app on node. + :param node: Topology node. + :param path: Nginx install path. + :param filename: Nginx conf name. + :type node: dict + :type path: str + :type filename: str + + :returns: nothing + """ + cmd_options = NginxUtil.get_cmd_options(path=path, + filename=filename) + + exec_cmd_no_error(node, cmd_options, sudo=True, disconnect=True, + message=u"Nginx start failed!") + + @staticmethod + def nginx_config_verify(node, path): + """Start NGINX cmd app on node. + :param node: Topology node. + :param path: Nginx install path. + :type node: dict + :type path: str + :returns: nothing + """ + cmd_options = NginxUtil.get_cmd_options(path=path) + exec_cmd_no_error(node, cmd_options, sudo=True, disconnect=True, + message=u"Nginx Config failed!") + + @staticmethod + def taskset_nginx_pid_to_idle_cores(node, cpu_idle_list): + """Set idle cpus to NGINX pid on node. + + :param node: Topology node. + :param cpu_idle_list: Idle Cpus. + :type node: dict + :type cpu_idle_list: list + :returns: nothing + """ + if node[u"type"] != NodeType.DUT: + raise RuntimeError(u'Node type is not a DUT!') + pids = DUTSetup.get_pid(node, u"nginx") + for index, pid in enumerate(pids): + cmd = f"taskset -pc {cpu_idle_list[index]} {pid}" + exec_cmd_no_error( + node, cmd, sudo=True, timeout=180, + message=u"taskset cores to nginx pid failed!" + ) diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py index e670b692de..fd0d8cfee0 100644 --- a/resources/libraries/python/autogen/Regenerator.py +++ b/resources/libraries/python/autogen/Regenerator.py @@ -85,7 +85,7 @@ def get_iface_and_suite_ids(filename): # It was something like "2n1l", we need one more split. dash_split = dash_split[1].split(u"-", 1) nic_code = dash_split[0] - suite_id = dash_split[1].split(u".", 1)[0] + suite_id = dash_split[1].split(u".robot", 1)[0] suite_tag = suite_id.rsplit(u"-", 1)[0] for prefix in Constants.FORBIDDEN_SUITE_PREFIX_LIST: if suite_tag.startswith(prefix): @@ -553,6 +553,17 @@ class Regenerator: {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4} ] + http_kwargs_list = [ + {u"frame_size": 0, u"phy_cores": 1}, + {u"frame_size": 0, u"phy_cores": 2}, + {u"frame_size": 64, u"phy_cores": 1}, + {u"frame_size": 64, u"phy_cores": 2}, + {u"frame_size": 1024, u"phy_cores": 1}, + {u"frame_size": 1024, u"phy_cores": 2}, + {u"frame_size": 2048, u"phy_cores": 1}, + {u"frame_size": 2048, u"phy_cores": 2} + ] + for in_filename in glob(pattern): if not self.quiet: print( @@ -583,6 +594,9 @@ class Regenerator: ) elif in_filename.endswith(u"-reconf.robot"): write_reconf_files(in_filename, in_prolog, default_kwargs_list) + elif in_filename.endswith(u"-rps.robot") \ + or in_filename.endswith(u"-cps.robot"): + write_tcp_files(in_filename, in_prolog, http_kwargs_list) elif in_filename.endswith(u"-bps.robot"): hoststack_kwargs_list = \ hs_quic_kwargs_list if u"quic" in in_filename \ diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py index 173c5919af..643d32a3cb 100644 --- a/resources/libraries/python/autogen/Testcase.py +++ b/resources/libraries/python/autogen/Testcase.py @@ -100,7 +100,14 @@ class Testcase: # TODO: Choose a better frame size identifier for streamed protocols # (TCP, QUIC, SCTP, ...) where DUT (not TG) decides frame size. if u"tcphttp" in suite_id: - template_string = f''' + if u"rps" or u"cps" in suite_id: + template_string = f''' +| ${{frame_str}}-${{cores_str}}c-{suite_id} +| | [Tags] | ${{frame_str}} | ${{cores_str}}C +| | frame_size=${{frame_num}} | phy_cores=${{cores_num}} +''' + else: + template_string = f''' | IMIX-${{cores_str}}c-{suite_id} | | [Tags] | ${{cores_str}}C | | phy_cores=${{cores_num}} diff --git a/resources/libraries/robot/hoststack/hoststack.robot b/resources/libraries/robot/hoststack/hoststack.robot index 075cc2b8bf..30363f9b91 100644 --- a/resources/libraries/robot/hoststack/hoststack.robot +++ b/resources/libraries/robot/hoststack/hoststack.robot @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -15,10 +15,13 @@ | Library | resources.libraries.python.InterfaceUtil | Library | resources.libraries.python.IPUtil | Library | resources.libraries.python.HoststackUtil +| Library | resources.libraries.python.NginxUtil | Library | resources.libraries.python.NsimUtil +| Library | resources.tools.ab.ABTools | Variables | resources/libraries/python/Constants.py | Resource | resources/libraries/robot/ip/ip4.robot | Resource | resources/libraries/robot/nsim/nsim.robot +| Resource | resources/libraries/robot/nginx/default.robot | | Documentation | L2 keywords to set up VPP to test hoststack. @@ -106,6 +109,16 @@ | ... | ip_address=${EMPTY} | ... | parallel=${1} | ... | time=${20} +| &{nginx_server_attr}= +| ... | role=server +| ... | cpu_cnt=${1} +| ... | cfg_vpp_feature=${Empty} +| ... | namespace=default +| ... | vcl_config=vcl_iperf3.conf +| ... | ld_preload=${True} +| ... | transparent_tls=${False} +| ... | json=${True} +| ... | ip_version=${4} *** Keywords *** | Set VPP Hoststack Attributes @@ -539,3 +552,72 @@ | | ... | ${vpp_nsim_attr} | ${iperf3_client} | | Then Set test message | ${client_output} | | Return From Keyword | ${client_defer_fail} + +| Set up LDP or VCL Nginx on DUT node +| | [Documentation] +| | ... | Setup for suites which uses VCL or LDP Nginx on DUT. +| | +| | ... | *Arguments:* +| | ... | - dut - DUT node. +| | ... | Type: string +| | ... | - mode - VCL Nginx or LDP Nginx. +| | ... | Type: string +| | ... | - rps_cps - Test request or connect. +| | ... | Type: string +| | ... | - core_num - Nginx work processes number. +| | ... | Type: int +| | ... | - qat - Whether to use the qat engine. +| | ... | Type: string +| | ... | - tls_tcp - TLS or TCP. +| | +| | ... | *Example:* +| | +| | ... | \| Set up LDP or VCL NGINX on DUT node \| ${dut} |${mode}\ +| | ... | \| ${rps_cps} \| ${phy_cores} \| ${qat} \| ${tls_tcp} \| +| | +| | [Arguments] | ${dut} | ${mode} | ${rps_cps} | ${phy_cores} | ${qat} +| | ... | ${tls_tcp} +| | +| | Set Interface State | ${dut} | ${DUT1_${int}1}[0] | up +| | VPP Interface Set IP Address | ${dut} | ${DUT1_${int}1}[0] +| | ... | ${dut_ip_addrs}[0] | ${dut_ip_prefix} +| | Vpp Node Interfaces Ready Wait | ${dut} +| | ${skip_cnt}= | Evaluate +| | ... | ${CPU_CNT_SYSTEM} + ${CPU_CNT_MAIN} + ${vpp_hoststack_attr}[phy_cores] +| | ${numa}= | Get interfaces numa node | ${dut} | ${DUT1_${int}1}[0] +| | Apply Nginx configuration on DUT | ${dut} | ${phy_cores} +| | Set To Dictionary | ${nginx_server_attr} | ip_address +| | ... | ${dut_ip_addrs}[0] +| | ${core_list}= | Cpu list per node str | ${dut} | ${numa} +| | ... | skip_cnt=${skip_cnt} | cpu_cnt=${nginx_server_attr}[cpu_cnt] +| | ${cpu_idle_list}= | Get cpu idle list | ${dut} | ${numa} +| | ... | ${smt_used} | ${cpu_alloc_str} +| | ${nginx_server}= | Get Nginx Command | ${nginx_server_attr} +| | ... | ${nginx_version} | ${packages_dir} +| | ${server_pid}= | Start Hoststack Test Program +| | ... | ${dut} | ${nginx_server_attr}[namespace] | ${core_list} +| | ... | ${nginx_server} +| | Taskset Nginx PID to idle cores | ${dut} | ${cpu_idle_list} + +| Measure TLS requests or connections per second +| | [Documentation] +| | ... | Measure number of requests or connections per second using ab. +| | +| | ... | *Arguments:* +| | ... | - ${ciphers} - Specify SSL/TLS cipher suite +| | ... | - ${files} - Filename to be requested from the servers +| | ... | - ${tls_tcp} - Test TLS or TCP. +| | ... | - ${mode} - VCL Nginx or LDP Nginx. +| | +| | ... | *Example:* +| | +| | ... | \| Measure TLS requests or connections per second +| | ... | \| AES128-SHA \| 64 \| tls \| rps \| +| | +| | [Arguments] | ${ciphers} | ${files} | ${tls_tcp} | ${mode} +| | +| | ${output}= | Run ab | ${tg} | ${dut_ip_addrs}[0] | ${ab_ip_addrs}[0] +| | ... | ${tls_tcp} | ${ciphers} | ${files} | ${mode} | ${r_total} | ${c_total} +| | ... | ${listen_port} +| | Set test message | ${output} +| | Log VPP Hoststack data | ${dut1} diff --git a/resources/libraries/robot/nginx/default.robot b/resources/libraries/robot/nginx/default.robot new file mode 100644 index 0000000000..5126da5858 --- /dev/null +++ b/resources/libraries/robot/nginx/default.robot @@ -0,0 +1,61 @@ +# Copyright (c) 2021 Intel and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Library | resources.libraries.python.InterfaceUtil +| Library | resources.libraries.python.NginxConfigGenerator +| Library | Collections + +*** Keywords *** +| Apply Nginx configuration on DUT +| | [Documentation] +| | ... | Setup for suites which uses VCL or LDP Nginx on DUT. +| | +| | ... | *Arguments:* +| | ... | - dut - DUT node. +| | ... | Type: string +| | ... | - phy_cores - vpp used phy cores number. +| | +| | ... | *Example:* +| | +| | ... | \| Apply Nginx configuration on DUT \| ${dut} | ${phy_cores} +| | +| | [Arguments] | ${dut} | ${phy_cores} +| | +| | Import Library | resources.libraries.python.NginxConfigGenerator +| | ... | WITH NAME | nc_manager +| | Run Keyword | nc_manager.Set Node | ${dut} +| | Run Keyword | nc_manager.Set Nginx Path | ${packages_dir} | ${nginx_version} +| | Run Keyword | nc_manager.Add Worker Processes | ${phy_cores} | ${smt_used} +| | Run Keyword | nc_manager.Add Master Process +| | Run Keyword | nc_manager.Add Daemon +| | Run Keyword | nc_manager.Add Worker Rlimit Nofile +| | Run Keyword | nc_manager.Add Events Use +| | Run Keyword | nc_manager.Add Events Worker Connections +| | Run Keyword | nc_manager.Add Events Accept Mutex +| | Run Keyword | nc_manager.Add Events Multi Accept +| | Run Keyword | nc_manager.Add Http Access Log +| | Run Keyword | nc_manager.Add Http Include +| | Run Keyword | nc_manager.Add Http Default Type +| | Run Keyword | nc_manager.Add Http Sendfile +| | Run Keyword | nc_manager.Add Http Keepalive Timeout | ${keep_time} +| | Run Keyword If | ${keep_time} > 0 +| | ... | nc_manager.Add Http Keepalive Requests | ${r_total} +| | Run Keyword | nc_manager.Add Http Server Listen | ${listen_port} +| | Run Keyword | nc_manager.Add Http Server Root +| | Run Keyword | nc_manager.Add Http Server Index +| | Run Keyword | nc_manager.Add Http Server Location | ${0} +| | Run Keyword | nc_manager.Add Http Server Location | ${64} +| | Run Keyword | nc_manager.Add Http Server Location | ${1024} +| | Run Keyword | nc_manager.Add Http Server Location | ${2048} +| | Run Keyword | nc_manager.Apply Config
\ No newline at end of file diff --git a/resources/libraries/robot/shared/suite_setup.robot b/resources/libraries/robot/shared/suite_setup.robot index a9e57e7024..09cec67e4e 100644 --- a/resources/libraries/robot/shared/suite_setup.robot +++ b/resources/libraries/robot/shared/suite_setup.robot @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,6 +16,8 @@ *** Settings *** | Library | resources.libraries.python.DPDK.DPDKTools | Library | resources.libraries.python.InterfaceUtil +| Library | resources.libraries.python.NGINX.NGINXTools +| Library | resources.tools.ab.ABTools | Library | resources.libraries.python.NodePath | Library | resources.libraries.python.topology.Topology | Library | resources.libraries.python.TrafficGenerator @@ -198,3 +200,37 @@ | | Configure crypto device on all DUTs | ${crypto_type} | numvfs=${numvfs} | | ... | force_init=${True} | | Configure kernel module on all DUTs | vfio_pci | force_load=${True} + +| Additional Suite Setup Action For nginx +| | [Documentation] +| | ... | Additional Setup for suites which uses Nginx. +| | +| | Install NGINX framework on all DUTs | ${nodes} | ${packages_dir} +| | ... | ${nginx_version} + +| Additional Suite Setup Action For ab +| | [Documentation] +| | ... | Additional Setup for suites which uses ab TG. +| | +| | Verify Program Installed | ${tg} | ab +| | Iface update numa node | ${tg} +| | ${running}= | Is TRex running | ${tg} +| | Run keyword if | ${running}==${True} | Teardown traffic generator | ${tg} +| | ${curr_driver}= | Get PCI dev driver | ${tg} +| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} +| | Run keyword if | '${curr_driver}'!='${None}' +| | ... | PCI Driver Unbind | ${tg} | +| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} +| | ${driver}= | Get Variable Value | ${tg['interfaces']['${tg_if1}']['driver']} +| | PCI Driver Bind | ${tg} +| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} | ${driver} +| | ${intf_name}= | Get Linux interface name | ${tg} +| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} +| | FOR | ${ip_addr} | IN | @{ab_ip_addrs} +| | | ${ip_addr_on_intf}= | Linux interface has IP | ${tg} | ${intf_name} +| | | ... | ${ip_addr} | ${ab_ip_prefix} +| | | Run Keyword If | ${ip_addr_on_intf}==${False} | Set Linux interface IP +| | | ... | ${tg} | ${intf_name} | ${ip_addr} | ${ab_ip_prefix} +| | END +| | Set Linux interface up | ${tg} | ${intf_name} +| | Check ab | ${tg} diff --git a/resources/libraries/robot/shared/suite_teardown.robot b/resources/libraries/robot/shared/suite_teardown.robot index f164b0eeb5..20b2776eec 100644 --- a/resources/libraries/robot/shared/suite_teardown.robot +++ b/resources/libraries/robot/shared/suite_teardown.robot @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -35,6 +35,19 @@ | | END | | Remove All Added VIF Ports On All DUTs From Topology | ${nodes} +| Additional Suite Tear Down Action For ab +| | [Documentation] +| | ... | Additional teardown for suites which uses ab. +| | +| | ${intf_name}= | Get Linux interface name | ${tg} +| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} +| | FOR | ${ip_addr} | IN | @{ab_ip_addrs} +| | | ${ip_addr_on_intf}= | Linux Interface Has IP | ${tg} | ${intf_name} +| | | ... | ${ip_addr} | ${ab_ip_prefix} +| | | Run Keyword If | ${ip_addr_on_intf}==${True} | Delete Linux Interface IP +| | | ... | ${tg} | ${intf_name} | ${ip_addr} | ${ab_ip_prefix} +| | END + | Additional Suite Tear Down Action For performance | | [Documentation] | | ... | Additional teardown for suites which uses performance measurement. diff --git a/resources/libraries/robot/shared/test_teardown.robot b/resources/libraries/robot/shared/test_teardown.robot index 18be67cfc5..977a87d5a6 100644 --- a/resources/libraries/robot/shared/test_teardown.robot +++ b/resources/libraries/robot/shared/test_teardown.robot @@ -86,6 +86,14 @@ | | | Destroy all '${container_group}' containers | | END +| Additional Test Tear Down Action For nginx +| | [Documentation] +| | ... | Additional teardown for tests which uses nginx. +| | +| | FOR | ${dut} | IN | @{duts} +| | | Kill Program | ${nodes['${dut}']} | nginx +| | END + | Additional Test Tear Down Action For det44 | | [Documentation] | | ... | Additional teardown for tests which uses DET44 feature. |