perf: add TCP Nginx+LDPRELOAD suites

1. Suite steup add download nginx
2. Add nginx-1.14.2/1.15.0 ldp test suite
3. Add NginxUtils,NginxConfigGenerator method
4. Taskset the PID of nginx to the unused cores in VPP and these cores are under NIC's NUMA ID
5. cleanup add Kill Processes - nohup

Signed-off-by: xizhanx <xix.zhang@intel.com>
Change-Id: Idbf0e4ec3bf63e88281a8e3e34f52e00a6801c85
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

19 files changed, 1455 insertions, 13 deletions

diff --git a/resources/libraries/bash/entry/check/tc_naming.sh b/resources/libraries/bash/entry/check/tc_naming.sh
index e9f86fc0dd..bc2ac32671 100644
--- a/resources/libraries/bash/entry/check/tc_naming.sh
+++ b/resources/libraries/bash/entry/check/tc_naming.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -48,14 +48,14 @@ r_parse='(.*)\/(.*).robot.*(([0-9]{2,4}B|IMIX)-.*)'
 # One caveat of this solution is that we cannot proceed to check full names now
 # as majority of Testcases does not meet naming criteria.
 s_testc_rules=(
-    'packet size'
+    'packet size or file size'
     'core combination'
     'NIC driver mode'
     'packet encapsulation on L2 layer'
     'test type'
     )
 r_testc_rules=(
-    '^([[:digit:]]{2,4}B|IMIX)-'
+    '^([[:digit:]]{1,4}B|IMIX)-'
     '([[:digit:]]+c-){0,1}'
     '(avf-|1lbvpplacp-|2lbvpplacp-){0,1}'
     '(eth|dot1q|dot1ad)'
diff --git a/resources/libraries/bash/entry/install_nginx.sh b/resources/libraries/bash/entry/install_nginx.sh
new file mode 100755
index 0000000000..3a2c8ef7eb
--- /dev/null
+++ b/resources/libraries/bash/entry/install_nginx.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install the NGINX framework on the DUT node. Check prerequisites.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+  echo "Source failed." >&2
+  exit 1
+}
+source "${BASH_FUNCTION_DIR}/nginx.sh" || die "Source failed."
+common_dirs ${@} || die
+gather_nginx  || die "Download nginx failed."
+nginx_extract || die "Extract nginx failed."
+nginx_compile || die "Compile nginx failed."
diff --git a/resources/libraries/bash/function/nginx.sh b/resources/libraries/bash/function/nginx.sh
new file mode 100755
index 0000000000..122af23852
--- /dev/null
+++ b/resources/libraries/bash/function/nginx.sh
@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+
+function gather_nginx () {
+
+    # Ensure stable NGINX archive is downloaded.
+    #
+    # Variables read:
+    # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
+    # - NGINX_VER - Version number of Nginx.
+    set -exuo pipefail
+    pushd "${DOWNLOAD_DIR}" || die "Pushd failed."
+    nginx_repo="http://nginx.org/download/"
+    # Use downloaded packages with specific version
+    echo "Downloading NGINX package of specific version from repo ..."
+    # Downloading NGINX version based on what VPP is using. Currently
+    # it is not easy way to detect from VPP version automatically.
+    nginx_stable_ver="${NGINX_VER}".tar.gz
+
+    if [[ ! -f "${nginx_stable_ver}" ]]; then
+        wget -nv --no-check-certificate \
+        "${nginx_repo}/${nginx_stable_ver}"  || {
+            die "Failed to get NGINX package from: ${nginx_repo}."
+        }
+    fi
+    popd || die "Popd failed."
+}
+
+
+function common_dirs () {
+
+    # Set global variables, create some directories (without touching content).
+    # This function assumes running in remote testbed. It might override other
+    # functions if included from common.sh.
+
+    # Arguments:
+    # - ${1} - Version number of Nginx.
+    # Variables set:
+    # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
+    # - CSIT_DIR - Path to CSIT framework.
+    # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
+    # - NGINX_DIR - Path to NGINX framework.
+    # - NGINX_VER - Version number of Nginx.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+    NGINX_VER="${1}"
+    this_file=$(readlink -e "${BASH_SOURCE[0]}") || {
+        die "Some error during locating of this source file."
+    }
+    BASH_FUNCTION_DIR=$(dirname "${this_file}") || {
+        die "Some error during dirname call."
+    }
+    CSIT_DIR=$(readlink -e "/tmp/openvpp-testing") || {
+        die "Readlink failed."
+    }
+    DOWNLOAD_DIR=$(readlink -f "${CSIT_DIR}/download_dir") || {
+        die "Readlink failed."
+    }
+    mkdir -p "${CSIT_DIR}/${NGINX_VER}" || die "Mkdir failed."
+    NGINX_DIR=$(readlink -e "${CSIT_DIR}/${NGINX_VER}") || {
+        die "Readlink failed."
+    }
+}
+
+
+
+function nginx_compile () {
+
+    # Compile NGINX archive.
+    #
+    # Variables read:
+    # - NGINX_DIR - Path to NGINX framework.
+    # - CSIT_DIR - Path to CSIT framework.
+    # - NGINX_INS_PATH - Path to NGINX install path.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+    NGINX_INS_PATH="${DOWNLOAD_DIR}/${NGINX_VER}"
+    pushd "${NGINX_DIR}" || die "Pushd failed."
+
+    # Set installation prefix.
+    param="--prefix=${NGINX_INS_PATH} "
+    # Set nginx binary pathname.
+    param+="--sbin-path=${NGINX_INS_PATH}/sbin/nginx "
+    # Set nginx.conf pathname.
+    param+="--conf-path=${NGINX_INS_PATH}/conf/nginx.conf "
+    # Enable ngx_http_stub_status_module.
+    param+="--with-http_stub_status_module "
+    # Force PCRE library usage.
+    param+="--with-pcre "
+    # Enable ngx_http_realip_module.
+    param+="--with-http_realip_module "
+    params=(${param})
+    ./configure "${params[@]}" || die "Failed to configure NGINX!"
+    make -j 16;make install || die "Failed to compile NGINX!"
+}
+
+
+function nginx_extract () {
+
+    # Extract NGINX framework.
+    #
+    # Variables read:
+    # - NGINX_DIR - Path to NGINX framework.
+    # - CSIT_DIR - Path to CSIT framework.
+    # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
+    # - NGINX_VER - Version number of Nginx.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    pushd "${CSIT_DIR}" || die "Pushd failed."
+    tar -xvf ${DOWNLOAD_DIR}/${NGINX_VER}.tar.gz --strip=1 \
+          --directory "${NGINX_DIR}" || {
+          die "Failed to extract NGINX!"
+    }
+}
diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py
index 79b94be7ff..be9fe34915 100644
--- a/resources/libraries/python/Constants.py
+++ b/resources/libraries/python/Constants.py
@@ -130,6 +130,9 @@ class Constants:
     # shell scripts location
     RESOURCES_LIB_SH = u"resources/libraries/bash"
 
+    # python scripts location
+    RESOURCES_LIB_PY = u"resources/libraries/python"
+
     # Python API provider location
     RESOURCES_PAPI_PROVIDER = u"resources/tools/papi/vpp_papi_provider.py"
 
diff --git a/resources/libraries/python/CpuUtils.py b/resources/libraries/python/CpuUtils.py
index 170cbe6b2e..293d6b6913 100644
--- a/resources/libraries/python/CpuUtils.py
+++ b/resources/libraries/python/CpuUtils.py
@@ -255,7 +255,7 @@ class CpuUtils:
             cpu_list_0 = cpu_list[:cpu_list_len // CpuUtils.NR_OF_THREADS]
             cpu_list_1 = cpu_list[cpu_list_len // CpuUtils.NR_OF_THREADS:]
             cpu_range = f"{cpu_list_0[0]}{sep}{cpu_list_0[-1]}," \
-                f"{cpu_list_1[0]}{sep}{cpu_list_1[-1]}"
+                        f"{cpu_list_1[0]}{sep}{cpu_list_1[-1]}"
         else:
             cpu_range = f"{cpu_list[0]}{sep}{cpu_list[-1]}"
 
@@ -469,3 +469,25 @@ class CpuUtils:
         return CpuUtils.cpu_slice_of_list_per_node(
             node, cpu_node=cpu_node, skip_cnt=skip_cnt, cpu_cnt=cpu_cnt,
             smt_used=False)
+
+    @staticmethod
+    def get_cpu_idle_list(node, cpu_node, smt_used, cpu_alloc_str, sep=u","):
+        """
+        Get idle CPU List
+        :param node: Node dictionary with cpuinfo.
+        :param cpu_node: Numa node number.
+        :param smt_used: True - we want to use SMT, otherwise false.
+        :param cpu_alloc_str: vpp used cores.
+        :param sep: Separator, default: ",".
+        :type node: dict
+        :type cpu_node: int
+        :type smt_used: bool
+        :type cpu_alloc_str: str
+        :type smt_used: bool
+        :type sep: str
+        :rtype: list
+        """
+        cpu_list = CpuUtils.cpu_list_per_node(node, cpu_node, smt_used)
+        cpu_idle_list = [i for i in cpu_list
+                         if str(i) not in cpu_alloc_str.split(sep)]
+        return cpu_idle_list
diff --git a/resources/libraries/python/HoststackUtil.py b/resources/libraries/python/HoststackUtil.py
index c307946698..e797c3c206 100644
--- a/resources/libraries/python/HoststackUtil.py
+++ b/resources/libraries/python/HoststackUtil.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -165,6 +165,39 @@ class HoststackUtil():
         return stdout_log, stderr_log
 
     @staticmethod
+    def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
+        """Construct the NGINX command using the specified attributes.
+
+        :param nginx_attributes: NGINX test program attributes.
+        :param nginx_version: NGINX version.
+        :param nginx_ins_dir: NGINX install dir.
+        :type nginx_attributes: dict
+        :type nginx_version: str
+        :type nginx_ins_dir: str
+        :returns: Command line components of the NGINX command
+            'env_vars' - environment variables
+            'name' - program name
+            'args' - command arguments.
+            'path' - program path.
+        :rtype: dict
+        """
+        nginx_cmd = dict()
+        nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
+                                 f"{Constants.RESOURCES_TPL_VCL}/" \
+                                 f"{nginx_attributes[u'vcl_config']}"
+        if nginx_attributes[u"ld_preload"]:
+            nginx_cmd[u"env_vars"] += \
+                f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
+        if nginx_attributes[u'transparent_tls']:
+            nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
+
+        nginx_cmd[u"name"] = u"nginx"
+        nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
+        nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
+                             f"nginx-{nginx_version}/conf/nginx.conf"
+        return nginx_cmd
+
+    @staticmethod
     def start_hoststack_test_program(node, namespace, core_list, program):
         """Start the specified HostStack test program.
 
@@ -194,9 +227,13 @@ class HoststackUtil():
 
         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
         args = program[u"args"]
-        cmd = f"nohup {shell_cmd} \'{env_vars}taskset --cpu-list {core_list} " \
-            f"{program_name} {args} >/tmp/{program_name}_stdout.log " \
-            f"2>/tmp/{program_name}_stderr.log &\'"
+        program_path = program.get(u"path", u"")
+        # NGINX used `worker_cpu_affinity` in configuration file
+        taskset_cmd = u"" if program_name == u"nginx" else \
+                                             f"taskset --cpu-list {core_list}"
+        cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
+              f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
+              f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
         try:
             exec_cmd_no_error(node, cmd, sudo=True)
             return DUTSetup.get_pid(node, program_name)[0]
@@ -350,3 +387,18 @@ class HoststackUtil():
         :rtype: bool
         """
         return server_defer_fail and client_defer_fail
+
+    @staticmethod
+    def log_vpp_hoststack_data(node):
+        """Retrieve and log VPP HostStack data.
+
+        :param node: DUT node.
+        :type node: dict
+        :raises RuntimeError: If node subtype is not a DUT or startup failed.
+        """
+
+        if node[u"type"] != u"DUT":
+            raise RuntimeError(u"Node type is not a DUT!")
+
+        PapiSocketExecutor.run_cli_cmd(node, u"show error")
+        PapiSocketExecutor.run_cli_cmd(node, u"show interface")
diff --git a/resources/libraries/python/NGINX/NGINXTools.py b/resources/libraries/python/NGINX/NGINXTools.py
new file mode 100644
index 0000000000..9418484f15
--- /dev/null
+++ b/resources/libraries/python/NGINX/NGINXTools.py
@@ -0,0 +1,145 @@
+# Copyright (c) 2021 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""This module implements initialization and cleanup of NGINX framework."""
+
+from robot.api import logger
+
+from resources.libraries.python.Constants import Constants
+from resources.libraries.python.ssh import exec_cmd_no_error, exec_cmd
+from resources.libraries.python.topology import NodeType
+from resources.libraries.python.NginxUtil import NginxUtil
+
+
+class NGINXTools:
+    """This class implements:
+    - Initialization of NGINX environment,
+    - Cleanup of NGINX environment.
+    """
+
+    @staticmethod
+    def cleanup_nginx_framework(node, nginx_ins_path):
+        """
+        Cleanup the NGINX framework on the DUT node.
+
+        :param node: Will cleanup the nginx on this nodes.
+        :param nginx_ins_path: NGINX install path.
+        :type node: dict
+        :type nginx_ins_path: str
+        :raises RuntimeError: If it fails to cleanup the nginx.
+        """
+        check_path_cmd = NginxUtil.get_cmd_options(path=nginx_ins_path)
+        exec_cmd_no_error(node, check_path_cmd, timeout=180,
+                          message=u"Check NGINX install path failed!")
+        command = f"rm -rf {nginx_ins_path}"
+        message = u"Cleanup the NGINX failed!"
+        exec_cmd_no_error(node, command, timeout=180, message=message)
+
+    @staticmethod
+    def cleanup_nginx_framework_on_all_duts(nodes, nginx_ins_path):
+        """
+        Cleanup the NGINX framework on all DUT nodes.
+
+        :param nodes: Will cleanup the nginx on this nodes.
+        :param nginx_ins_path: NGINX install path.
+        :type nodes: dict
+        :type nginx_ins_path: str
+        :raises RuntimeError: If it fails to cleanup the nginx.
+        """
+        for node in nodes.values():
+            if node[u"type"] == NodeType.DUT:
+                NGINXTools.cleanup_nginx_framework(node, nginx_ins_path)
+
+    @staticmethod
+    def install_original_nginx_framework(node, pkg_dir, nginx_version):
+        """
+        Prepare the NGINX framework on the DUT node.
+
+        :param node: Node from topology file.
+        :param pkg_dir: Ldp NGINX install dir.
+        :param nginx_version: NGINX Version.
+        :type node: dict
+        :type pkg_dir: str
+        :type nginx_version: str
+        :raises RuntimeError: If command returns nonzero return code.
+        """
+        nginx_path = f"{pkg_dir}/nginx-{nginx_version}/sbin/nginx"
+        cmd_options = NginxUtil.get_cmd_options(path=nginx_path)
+        ret_code, _, stderr = exec_cmd(node, cmd_options, sudo=True)
+        if nginx_version in stderr and ret_code == 0:
+            logger.info(f"NGINX Version: {stderr}")
+            return
+        command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}" \
+                  f"/entry/install_nginx.sh nginx-{nginx_version}"
+        message = u"Install the NGINX failed!"
+        exec_cmd_no_error(node, command, sudo=True, timeout=600,
+                          message=message)
+        _, stderr = exec_cmd_no_error(node, cmd_options, sudo=True,
+                                      message=message)
+
+        logger.info(f"NGINX Version: {stderr}")
+
+    @staticmethod
+    def install_vsap_nginx_on_dut(node, pkg_dir):
+        """
+         Prepare the VSAP NGINX framework on all DUT
+
+        :param node: Node from topology file.
+        :param pkg_dir: Path to directory where packages are stored.
+        :type node: dict
+        :type pkg_dir: str
+        :raises RuntimeError: If command returns nonzero return code.
+        """
+        command = u". /etc/lsb-release; echo \"${DISTRIB_ID}\""
+        stdout, _ = exec_cmd_no_error(node, command)
+
+        if stdout.strip() == u"Ubuntu":
+            logger.console(u"NGINX install on DUT... ")
+            exec_cmd_no_error(
+                node, u"apt-get purge -y 'vsap*' || true", timeout=120,
+                sudo=True
+            )
+            exec_cmd_no_error(
+                node, f"dpkg -i --force-all {pkg_dir}vsap-nginx*.deb",
+                timeout=120, sudo=True,
+                message=u"Installation of vsap-nginx failed!"
+            )
+
+            exec_cmd_no_error(node, u"dpkg -l | grep vsap*",
+                              sudo=True)
+
+            logger.console(u"Completed!\n")
+        else:
+            logger.console(u"Ubuntu need!\n")
+
+    @staticmethod
+    def install_nginx_framework_on_all_duts(nodes, pkg_dir, nginx_version=None):
+        """
+        Prepare the NGINX framework on all DUTs.
+
+        :param nodes: Nodes from topology file.
+        :param pkg_dir: Path to directory where packages are stored.
+        :param nginx_version: NGINX version.
+        :type nodes: dict
+        :type pkg_dir: str
+        :type nginx_version: str
+        """
+
+        for node in list(nodes.values()):
+            if node[u"type"] == NodeType.DUT:
+                if nginx_version:
+                    NGINXTools.install_original_nginx_framework(node, pkg_dir,
+                                                                nginx_version)
+                else:
+                    NGINXTools.install_vsap_nginx_on_dut(node, pkg_dir)
diff --git a/resources/libraries/python/NGINX/__init__.py b/resources/libraries/python/NGINX/__init__.py
new file mode 100644
index 0000000000..d828cbe7cb
--- /dev/null
+++ b/resources/libraries/python/NGINX/__init__.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2021 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+__init__ file for directory resources/libraries/python/NGINX
+"""
diff --git a/resources/libraries/python/NginxConfigGenerator.py b/resources/libraries/python/NginxConfigGenerator.py
new file mode 100644
index 0000000000..1a0f5f077a
--- /dev/null
+++ b/resources/libraries/python/NginxConfigGenerator.py
@@ -0,0 +1,244 @@
+# Copyright (c) 2021 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Nginx Configuration File Generator library.
+"""
+
+from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.topology import NodeType
+from resources.libraries.python.NginxUtil import NginxUtil
+
+__all__ = [u"NginxConfigGenerator"]
+
+
+class NginxConfigGenerator:
+    """NGINX Configuration File Generator."""
+
+    def __init__(self):
+        """Initialize library."""
+        # VPP Node to apply configuration on
+        self._node = u""
+        # NGINX Startup config location
+        self._nginx_path = u"/usr/local/nginx/"
+        # Serialized NGinx Configuration
+        self._nginx_config = u""
+        # VPP Configuration
+        self._nodeconfig = dict()
+
+    def set_node(self, node):
+        """Set DUT node.
+
+        :param node: Node to store configuration on.
+        :type node: dict
+        :raises RuntimeError: If Node type is not DUT.
+        """
+        if node[u"type"] != NodeType.DUT:
+            raise RuntimeError(
+                u"Startup config can only be applied to DUTnode."
+            )
+        self._node = node
+
+    def set_nginx_path(self, packages_dir, nginx_version):
+        """Set NGINX Conf Name.
+
+        :param packages_dir: NGINX install path.
+        :param nginx_version: Test NGINX version.
+        :type packages_dir: str
+        :type nginx_version: str
+        :raises RuntimeError: If Node type is not DUT.
+        """
+        if nginx_version:
+            self._nginx_path = f"{packages_dir}/nginx-{nginx_version}"
+
+    def add_http_server_listen(self, value):
+        """Add Http Server listen port configuration."""
+        path = [u"http", u"server", u"listen"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_http_server_root(self, value=u"html"):
+        """Add Http Server root configuration."""
+        path = [u"http", u"server", u"root"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_http_server_index(self, value=u"index.html index.htm"):
+        """Add Http Server index configuration."""
+        path = [u"http", u"server", u"index"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_config_item(self, config, value, path):
+        """Add NGINX configuration item.
+
+        :param config: Startup configuration of node.
+        :param value: Value to insert.
+        :param path: Path where to insert item.
+        :type config: dict
+        :type value: str
+        :type path: list
+        """
+        if len(path) == 1:
+            config[path[0]] = value
+            return
+        if path[0] not in config:
+            config[path[0]] = dict()
+        elif isinstance(config[path[0]], str):
+            config[path[0]] = dict() if config[path[0]] == u"" \
+                else {config[path[0]]: u""}
+        self.add_config_item(config[path[0]], value, path[1:])
+
+    def dump_config(self, obj, level=-1):
+        """Dump the startup configuration in NGINX config format.
+
+        :param obj: Python Object to print.
+        :param level: Nested level for indentation.
+        :type obj: Obj
+        :type level: int
+        :returns: nothing
+        """
+        indent = u"  "
+        if level >= 0:
+            self._nginx_config += f"{level * indent}{{\n"
+        if isinstance(obj, dict):
+            for key, val in obj.items():
+                if hasattr(val, u"__iter__") and not isinstance(val, str):
+                    self._nginx_config += f"{(level + 1) * indent}{key}\n"
+                    self.dump_config(val, level + 1)
+                else:
+                    self._nginx_config += f"{(level + 1) * indent}" \
+                                          f"{key} {val};\n"
+        else:
+            for val in obj:
+                self._nginx_config += f"{(level + 1) * indent}{val};\n"
+        if level >= 0:
+            self._nginx_config += f"{level * indent}}}\n"
+
+    def write_config(self, filename=None):
+        """Generate and write NGINX startup configuration to file.
+
+        :param filename: NGINX configuration file name.
+        :type filename: str
+        """
+        if filename is None:
+            filename = f"{self._nginx_path}/conf/nginx.conf"
+        self.dump_config(self._nodeconfig)
+        cmd = f"echo \"{self._nginx_config}\" | sudo tee {filename}"
+        exec_cmd_no_error(
+            self._node, cmd, message=u"Writing config file failed!"
+        )
+
+    def add_http_server_location(self, size):
+        """Add Http Server location configuration.
+
+        :param size: File size.
+        :type size: int
+        """
+        if size == 0:
+            files = u"return"
+        elif size >= 1024:
+            files = f"{int(size / 1024)}KB.json"
+        else:
+            files = f"{size}B.json"
+        key = f"{files}"
+        size_str = size * u"x"
+        value = "200 '%s'" % size_str
+        path = [u"http", u"server", f"location /{key}", u"return"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_http_access_log(self, value=u"off"):
+        """Add Http access_log configuration."""
+        path = [u"http", u"access_log"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_http_include(self, value=u"mime.types"):
+        """Add Http include configuration."""
+        path = [u"http", u"include"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_http_default_type(self, value=u"application/octet-stream"):
+        """Add Http default_type configuration."""
+        path = [u"http", u"default_type"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_http_sendfile(self, value=u"on"):
+        """Add Http sendfile configuration."""
+        path = [u"http", u"sendfile"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_http_keepalive_timeout(self, value):
+        """Add Http keepalive alive timeout configuration."""
+        path = [u"http", u"keepalive_timeout"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_http_keepalive_requests(self, value):
+        """Add Http keepalive alive requests configuration."""
+        path = [u"http", u"keepalive_requests"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_events_use(self, value=u"epoll"):
+        """Add Events use configuration."""
+        path = [u"events", u"use"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_events_worker_connections(self, value=10240):
+        """Add Events worker connections configuration."""
+        path = [u"events", u"worker_connections"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_events_accept_mutex(self, value=u"off"):
+        """Add Events accept mutex configuration."""
+        path = [u"events", u"accept_mutex"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_events_multi_accept(self, value=u"off"):
+        """Add Events multi accept configuration."""
+        path = [u"events", u"multi_accept"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_worker_rlimit_nofile(self, value=10240):
+        """Add Events worker rlimit nofile configuration."""
+        path = [u"worker_rlimit_nofile"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_master_process(self, value=u"on"):
+        """Add master process configuration."""
+        path = [u"master_process"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_daemon(self, value=u"off"):
+        """Add daemon configuration."""
+        path = [u"daemon"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def add_worker_processes(self, value, smt_used):
+        """Add worker processes configuration."""
+        # nginx workers : vpp used phy workers = 2:1
+        if smt_used:
+            value = value * 4
+        else:
+            value = value * 2
+        path = [u"worker_processes"]
+        self.add_config_item(self._nodeconfig, value, path)
+
+    def apply_config(self, filename=None, verify_nginx=True):
+        """Generate and write NGINX configuration to file and
+        verify configuration.
+
+        :param filename: NGINX configuration file name.
+        :param verify_nginx: Verify NGINX configuration.
+        :type filename: str
+        :type verify_nginx: bool
+        """
+        self.write_config(filename=filename)
+
+        app_path = f"{self._nginx_path}/sbin/nginx"
+        if verify_nginx:
+            NginxUtil.nginx_config_verify(self._node, app_path)
diff --git a/resources/libraries/python/NginxUtil.py b/resources/libraries/python/NginxUtil.py
new file mode 100644
index 0000000000..a19ac37291
--- /dev/null
+++ b/resources/libraries/python/NginxUtil.py
@@ -0,0 +1,124 @@
+# Copyright (c) 2021 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""NGINX Utilities Library."""
+
+from resources.libraries.python.OptionString import OptionString
+from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.topology import NodeType
+from resources.libraries.python.DUTSetup import DUTSetup
+
+
+class NginxUtil:
+    """Utilities for NGINX."""
+
+    @staticmethod
+    def get_cmd_options(**kwargs):
+        """Create parameters options.
+
+        :param kwargs: Dict of cmd parameters.
+        :type kwargs: dict
+        :returns: cmd parameters.
+        :rtype: OptionString
+        """
+        cmd_options = OptionString()
+        nginx_path = kwargs.get(u"path", u"/usr/local/nginx")
+        cmd_options.add(nginx_path)
+        options = OptionString(prefix=u"-")
+        # Show Nginx Version
+        options.add(u"v")
+        # Verify Configuration
+        options.add(u"t")
+        # Send signal to a master process: stop, quit, reopen.
+        options.add_with_value_from_dict(
+            u"s", u"signal", kwargs
+        )
+        # Set prefix path (default: /usr/local/nginx/).
+        options.add_with_value_from_dict(
+            u"p", u"prefix", kwargs
+        )
+        # Set configuration file (default: conf/nginx.conf).
+        options.add_with_value_from_dict(
+            u"c", u"filename", kwargs
+        )
+        # Set global directives out of configuration file
+        options.add_with_value_from_dict(
+            u"g", u"directives", kwargs
+        )
+        cmd_options.extend(options)
+        return cmd_options
+
+    @staticmethod
+    def nginx_cmd_stop(node, path):
+        """Stop NGINX cmd app on node.
+        :param node: Topology node.
+        :param path: Nginx install path.
+        :type node: dict
+        :type path: str
+        :returns: nothing
+        """
+        cmd_options = NginxUtil.get_cmd_options(path=path, signal=u"stop")
+
+        exec_cmd_no_error(node, cmd_options, sudo=True, disconnect=True,
+                          message=u"Nginx stop failed!")
+
+    @staticmethod
+    def nginx_cmd_start(node, path, filename):
+        """Start NGINX cmd app on node.
+        :param node: Topology node.
+        :param path: Nginx install path.
+        :param filename: Nginx conf name.
+        :type node: dict
+        :type path: str
+        :type filename: str
+
+        :returns: nothing
+        """
+        cmd_options = NginxUtil.get_cmd_options(path=path,
+                                                filename=filename)
+
+        exec_cmd_no_error(node, cmd_options, sudo=True, disconnect=True,
+                          message=u"Nginx start failed!")
+
+    @staticmethod
+    def nginx_config_verify(node, path):
+        """Start NGINX cmd app on node.
+        :param node: Topology node.
+        :param path: Nginx install path.
+        :type node: dict
+        :type path: str
+        :returns: nothing
+        """
+        cmd_options = NginxUtil.get_cmd_options(path=path)
+        exec_cmd_no_error(node, cmd_options, sudo=True, disconnect=True,
+                          message=u"Nginx Config failed!")
+
+    @staticmethod
+    def taskset_nginx_pid_to_idle_cores(node, cpu_idle_list):
+        """Set idle cpus to NGINX pid on node.
+
+        :param node: Topology node.
+        :param cpu_idle_list: Idle Cpus.
+        :type node: dict
+        :type cpu_idle_list: list
+        :returns: nothing
+        """
+        if node[u"type"] != NodeType.DUT:
+            raise RuntimeError(u'Node type is not a DUT!')
+        pids = DUTSetup.get_pid(node, u"nginx")
+        for index, pid in enumerate(pids):
+            cmd = f"taskset -pc {cpu_idle_list[index]} {pid}"
+            exec_cmd_no_error(
+                node, cmd, sudo=True, timeout=180,
+                message=u"taskset cores to nginx pid failed!"
+            )
diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
index e670b692de..fd0d8cfee0 100644
--- a/resources/libraries/python/autogen/Regenerator.py
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -85,7 +85,7 @@ def get_iface_and_suite_ids(filename):
         # It was something like "2n1l", we need one more split.
         dash_split = dash_split[1].split(u"-", 1)
     nic_code = dash_split[0]
-    suite_id = dash_split[1].split(u".", 1)[0]
+    suite_id = dash_split[1].split(u".robot", 1)[0]
     suite_tag = suite_id.rsplit(u"-", 1)[0]
     for prefix in Constants.FORBIDDEN_SUITE_PREFIX_LIST:
         if suite_tag.startswith(prefix):
@@ -553,6 +553,17 @@ class Regenerator:
             {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4}
         ]
 
+        http_kwargs_list = [
+            {u"frame_size": 0, u"phy_cores": 1},
+            {u"frame_size": 0, u"phy_cores": 2},
+            {u"frame_size": 64, u"phy_cores": 1},
+            {u"frame_size": 64, u"phy_cores": 2},
+            {u"frame_size": 1024, u"phy_cores": 1},
+            {u"frame_size": 1024, u"phy_cores": 2},
+            {u"frame_size": 2048, u"phy_cores": 1},
+            {u"frame_size": 2048, u"phy_cores": 2}
+        ]
+
         for in_filename in glob(pattern):
             if not self.quiet:
                 print(
@@ -583,6 +594,9 @@ class Regenerator:
                     )
             elif in_filename.endswith(u"-reconf.robot"):
                 write_reconf_files(in_filename, in_prolog, default_kwargs_list)
+            elif in_filename.endswith(u"-rps.robot") \
+                    or in_filename.endswith(u"-cps.robot"):
+                write_tcp_files(in_filename, in_prolog, http_kwargs_list)
             elif in_filename.endswith(u"-bps.robot"):
                 hoststack_kwargs_list = \
                     hs_quic_kwargs_list if u"quic" in in_filename \
diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py
index 173c5919af..643d32a3cb 100644
--- a/resources/libraries/python/autogen/Testcase.py
+++ b/resources/libraries/python/autogen/Testcase.py
@@ -100,7 +100,14 @@ class Testcase:
         # TODO: Choose a better frame size identifier for streamed protocols
         # (TCP, QUIC, SCTP, ...) where DUT (not TG) decides frame size.
         if u"tcphttp" in suite_id:
-            template_string = f'''
+            if u"rps" or u"cps" in suite_id:
+                template_string = f'''
+| ${{frame_str}}-${{cores_str}}c-{suite_id}
+| | [Tags] | ${{frame_str}} | ${{cores_str}}C
+| | frame_size=${{frame_num}} | phy_cores=${{cores_num}}
+'''
+            else:
+                template_string = f'''
 | IMIX-${{cores_str}}c-{suite_id}
 | | [Tags] | ${{cores_str}}C
 | | phy_cores=${{cores_num}}
diff --git a/resources/libraries/robot/hoststack/hoststack.robot b/resources/libraries/robot/hoststack/hoststack.robot
index 075cc2b8bf..30363f9b91 100644
--- a/resources/libraries/robot/hoststack/hoststack.robot
+++ b/resources/libraries/robot/hoststack/hoststack.robot
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -15,10 +15,13 @@
 | Library | resources.libraries.python.InterfaceUtil
 | Library | resources.libraries.python.IPUtil
 | Library | resources.libraries.python.HoststackUtil
+| Library | resources.libraries.python.NginxUtil
 | Library | resources.libraries.python.NsimUtil
+| Library | resources.tools.ab.ABTools
 | Variables | resources/libraries/python/Constants.py
 | Resource | resources/libraries/robot/ip/ip4.robot
 | Resource | resources/libraries/robot/nsim/nsim.robot
+| Resource | resources/libraries/robot/nginx/default.robot
 |
 | Documentation | L2 keywords to set up VPP to test hoststack.
 
@@ -106,6 +109,16 @@
 | ... | ip_address=${EMPTY}
 | ... | parallel=${1}
 | ... | time=${20}
+| &{nginx_server_attr}=
+| ... | role=server
+| ... | cpu_cnt=${1}
+| ... | cfg_vpp_feature=${Empty}
+| ... | namespace=default
+| ... | vcl_config=vcl_iperf3.conf
+| ... | ld_preload=${True}
+| ... | transparent_tls=${False}
+| ... | json=${True}
+| ... | ip_version=${4}
 
 *** Keywords ***
 | Set VPP Hoststack Attributes
@@ -539,3 +552,72 @@
 | | ... | ${vpp_nsim_attr} | ${iperf3_client}
 | | Then Set test message | ${client_output}
 | | Return From Keyword | ${client_defer_fail}
+
+| Set up LDP or VCL Nginx on DUT node
+| | [Documentation]
+| | ... | Setup for suites which uses VCL or LDP Nginx on DUT.
+| |
+| | ... | *Arguments:*
+| | ... | - dut - DUT node.
+| | ... | Type: string
+| | ... | - mode - VCL Nginx or LDP Nginx.
+| | ... | Type: string
+| | ... | - rps_cps - Test request or connect.
+| | ... | Type: string
+| | ... | - core_num - Nginx work processes number.
+| | ... | Type: int
+| | ... | - qat - Whether to use the qat engine.
+| | ... | Type: string
+| | ... | - tls_tcp - TLS or TCP.
+| |
+| | ... | *Example:*
+| |
+| | ... | \| Set up LDP or VCL NGINX on DUT node \| ${dut} |${mode}\
+| | ... | \| ${rps_cps} \| ${phy_cores} \| ${qat} \| ${tls_tcp} \|
+| |
+| | [Arguments] | ${dut} | ${mode} | ${rps_cps} | ${phy_cores} | ${qat}
+| | ... | ${tls_tcp}
+| |
+| | Set Interface State | ${dut} | ${DUT1_${int}1}[0] | up
+| | VPP Interface Set IP Address | ${dut} | ${DUT1_${int}1}[0]
+| | ... | ${dut_ip_addrs}[0] | ${dut_ip_prefix}
+| | Vpp Node Interfaces Ready Wait | ${dut}
+| | ${skip_cnt}= | Evaluate
+| | ... | ${CPU_CNT_SYSTEM} + ${CPU_CNT_MAIN} + ${vpp_hoststack_attr}[phy_cores]
+| | ${numa}= | Get interfaces numa node | ${dut} | ${DUT1_${int}1}[0]
+| | Apply Nginx configuration on DUT | ${dut} | ${phy_cores}
+| | Set To Dictionary | ${nginx_server_attr} | ip_address
+| | ... | ${dut_ip_addrs}[0]
+| | ${core_list}= | Cpu list per node str | ${dut} | ${numa}
+| | ... | skip_cnt=${skip_cnt} | cpu_cnt=${nginx_server_attr}[cpu_cnt]
+| | ${cpu_idle_list}= | Get cpu idle list | ${dut} | ${numa}
+| | ... | ${smt_used} | ${cpu_alloc_str}
+| | ${nginx_server}= | Get Nginx Command | ${nginx_server_attr}
+| | ... | ${nginx_version} | ${packages_dir}
+| | ${server_pid}= | Start Hoststack Test Program
+| | ... | ${dut} | ${nginx_server_attr}[namespace] | ${core_list}
+| | ... | ${nginx_server}
+| | Taskset Nginx PID to idle cores | ${dut} | ${cpu_idle_list}
+
+| Measure TLS requests or connections per second
+| | [Documentation]
+| | ... | Measure number of requests or connections per second using ab.
+| |
+| | ... | *Arguments:*
+| | ... | - ${ciphers} - Specify SSL/TLS cipher suite
+| | ... | - ${files} - Filename to be requested from the servers
+| | ... | - ${tls_tcp} - Test TLS or TCP.
+| | ... | - ${mode} - VCL Nginx or LDP Nginx.
+| |
+| | ... | *Example:*
+| |
+| | ... | \| Measure TLS requests or connections per second
+| | ... | \| AES128-SHA \| 64 \| tls \| rps \|
+| |
+| | [Arguments] | ${ciphers} | ${files} | ${tls_tcp} | ${mode}
+| |
+| | ${output}= | Run ab | ${tg} | ${dut_ip_addrs}[0] | ${ab_ip_addrs}[0]
+| | ... | ${tls_tcp} | ${ciphers} | ${files} | ${mode} | ${r_total} | ${c_total}
+| | ... | ${listen_port}
+| | Set test message | ${output}
+| | Log VPP Hoststack data | ${dut1}
diff --git a/resources/libraries/robot/nginx/default.robot b/resources/libraries/robot/nginx/default.robot
new file mode 100644
index 0000000000..5126da5858
--- /dev/null
+++ b/resources/libraries/robot/nginx/default.robot
@@ -0,0 +1,61 @@
+# Copyright (c) 2021 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Library | resources.libraries.python.InterfaceUtil
+| Library | resources.libraries.python.NginxConfigGenerator
+| Library | Collections
+
+*** Keywords ***
+| Apply Nginx configuration on DUT
+| | [Documentation]
+| | ... | Setup for suites which uses VCL or LDP Nginx on DUT.
+| |
+| | ... | *Arguments:*
+| | ... | - dut - DUT node.
+| | ... | Type: string
+| | ... | - phy_cores - vpp used phy cores number.
+| |
+| | ... | *Example:*
+| |
+| | ... | \| Apply Nginx configuration on DUT \| ${dut} | ${phy_cores}
+| |
+| | [Arguments] | ${dut} | ${phy_cores}
+| |
+| | Import Library | resources.libraries.python.NginxConfigGenerator
+| | ... | WITH NAME | nc_manager
+| | Run Keyword | nc_manager.Set Node | ${dut}
+| | Run Keyword | nc_manager.Set Nginx Path | ${packages_dir} | ${nginx_version}
+| | Run Keyword | nc_manager.Add Worker Processes | ${phy_cores} | ${smt_used}
+| | Run Keyword | nc_manager.Add Master Process
+| | Run Keyword | nc_manager.Add Daemon
+| | Run Keyword | nc_manager.Add Worker Rlimit Nofile
+| | Run Keyword | nc_manager.Add Events Use
+| | Run Keyword | nc_manager.Add Events Worker Connections
+| | Run Keyword | nc_manager.Add Events Accept Mutex
+| | Run Keyword | nc_manager.Add Events Multi Accept
+| | Run Keyword | nc_manager.Add Http Access Log
+| | Run Keyword | nc_manager.Add Http Include
+| | Run Keyword | nc_manager.Add Http Default Type
+| | Run Keyword | nc_manager.Add Http Sendfile
+| | Run Keyword | nc_manager.Add Http Keepalive Timeout | ${keep_time}
+| | Run Keyword If | ${keep_time} > 0
+| | ... | nc_manager.Add Http Keepalive Requests | ${r_total}
+| | Run Keyword | nc_manager.Add Http Server Listen | ${listen_port}
+| | Run Keyword | nc_manager.Add Http Server Root
+| | Run Keyword | nc_manager.Add Http Server Index
+| | Run Keyword | nc_manager.Add Http Server Location | ${0}
+| | Run Keyword | nc_manager.Add Http Server Location | ${64}
+| | Run Keyword | nc_manager.Add Http Server Location | ${1024}
+| | Run Keyword | nc_manager.Add Http Server Location | ${2048}
+| | Run Keyword | nc_manager.Apply Config
\ No newline at end of file
diff --git a/resources/libraries/robot/shared/suite_setup.robot b/resources/libraries/robot/shared/suite_setup.robot
index a9e57e7024..09cec67e4e 100644
--- a/resources/libraries/robot/shared/suite_setup.robot
+++ b/resources/libraries/robot/shared/suite_setup.robot
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -16,6 +16,8 @@
 *** Settings ***
 | Library | resources.libraries.python.DPDK.DPDKTools
 | Library | resources.libraries.python.InterfaceUtil
+| Library | resources.libraries.python.NGINX.NGINXTools
+| Library | resources.tools.ab.ABTools
 | Library | resources.libraries.python.NodePath
 | Library | resources.libraries.python.topology.Topology
 | Library | resources.libraries.python.TrafficGenerator
@@ -198,3 +200,37 @@
 | | Configure crypto device on all DUTs | ${crypto_type} | numvfs=${numvfs}
 | | ... | force_init=${True}
 | | Configure kernel module on all DUTs | vfio_pci | force_load=${True}
+
+| Additional Suite Setup Action For nginx
+| | [Documentation]
+| | ... | Additional Setup for suites which uses Nginx.
+| |
+| | Install NGINX framework on all DUTs | ${nodes} | ${packages_dir}
+| | ... |  ${nginx_version}
+
+| Additional Suite Setup Action For ab
+| | [Documentation]
+| | ... | Additional Setup for suites which uses ab TG.
+| |
+| | Verify Program Installed | ${tg} | ab
+| | Iface update numa node | ${tg}
+| | ${running}= | Is TRex running | ${tg}
+| | Run keyword if | ${running}==${True} | Teardown traffic generator | ${tg}
+| | ${curr_driver}= | Get PCI dev driver | ${tg}
+| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']}
+| | Run keyword if | '${curr_driver}'!='${None}'
+| | ... | PCI Driver Unbind | ${tg} |
+| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']}
+| | ${driver}= | Get Variable Value | ${tg['interfaces']['${tg_if1}']['driver']}
+| | PCI Driver Bind | ${tg}
+| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} | ${driver}
+| | ${intf_name}= | Get Linux interface name | ${tg}
+| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']}
+| | FOR | ${ip_addr} | IN | @{ab_ip_addrs}
+| | | ${ip_addr_on_intf}= | Linux interface has IP | ${tg} | ${intf_name}
+| | | ... | ${ip_addr} | ${ab_ip_prefix}
+| | | Run Keyword If | ${ip_addr_on_intf}==${False} | Set Linux interface IP
+| | | ... | ${tg} | ${intf_name} | ${ip_addr} | ${ab_ip_prefix}
+| | END
+| | Set Linux interface up | ${tg} | ${intf_name}
+| | Check ab | ${tg}
diff --git a/resources/libraries/robot/shared/suite_teardown.robot b/resources/libraries/robot/shared/suite_teardown.robot
index f164b0eeb5..20b2776eec 100644
--- a/resources/libraries/robot/shared/suite_teardown.robot
+++ b/resources/libraries/robot/shared/suite_teardown.robot
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -35,6 +35,19 @@
 | | END
 | | Remove All Added VIF Ports On All DUTs From Topology | ${nodes}
 
+| Additional Suite Tear Down Action For ab
+| | [Documentation]
+| | ... | Additional teardown for suites which uses ab.
+| |
+| | ${intf_name}= | Get Linux interface name | ${tg}
+| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']}
+| | FOR | ${ip_addr} | IN | @{ab_ip_addrs}
+| | | ${ip_addr_on_intf}= | Linux Interface Has IP | ${tg} | ${intf_name}
+| | | ... | ${ip_addr} | ${ab_ip_prefix}
+| | | Run Keyword If | ${ip_addr_on_intf}==${True} | Delete Linux Interface IP
+| | | ... | ${tg} | ${intf_name} | ${ip_addr} | ${ab_ip_prefix}
+| | END
+
 | Additional Suite Tear Down Action For performance
 | | [Documentation]
 | | ... | Additional teardown for suites which uses performance measurement.
diff --git a/resources/libraries/robot/shared/test_teardown.robot b/resources/libraries/robot/shared/test_teardown.robot
index 18be67cfc5..977a87d5a6 100644
--- a/resources/libraries/robot/shared/test_teardown.robot
+++ b/resources/libraries/robot/shared/test_teardown.robot
@@ -86,6 +86,14 @@
 | | | Destroy all '${container_group}' containers
 | | END
 
+| Additional Test Tear Down Action For nginx
+| | [Documentation]
+| | ... | Additional teardown for tests which uses nginx.
+| |
+| | FOR | ${dut} | IN | @{duts}
+| | | Kill Program | ${nodes['${dut}']} | nginx
+| | END
+
 | Additional Test Tear Down Action For det44
 | | [Documentation]
 | | ... | Additional teardown for tests which uses DET44 feature.
diff --git a/resources/tools/ab/ABFork.py b/resources/tools/ab/ABFork.py
new file mode 100755
index 0000000000..8436ed38be
--- /dev/null
+++ b/resources/tools/ab/ABFork.py
@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""ab fork library."""
+
+from multiprocessing import Pool
+import subprocess
+import argparse
+import re
+
+REGEX_RPS = r"Requests per second:\s*" \
+            r"(\d*\.*\S*)"
+REGEX_LATENCY = r"Time per request:\s*" \
+                r"(\d*\.*\S*)"
+REGEX_PROCESS = r"Time per request:\s*" \
+                r"(\d*\.*\S*)"
+REGEX_TR = r"Transfer rate:\s*" \
+           r"(\d*\.*\S*)"
+REGEX_TT = r"Total transferred:\s*" \
+           r"(\d*)"
+REGEX_OK_NUM = r"Complete requests:\s*" \
+               r"(\d*)"
+REGEX_FAILED_NUM = r"Failed requests:\s*" \
+                   r"(\d*)"
+REGEX_NUM = r"(\d*\.*\d*)(\D*)"
+
+
+def main():
+    """ main function. get option and run ab test.
+
+    :returns: Nothing.
+    """
+
+    # Get option.
+    parser = argparse.ArgumentParser(description=u"Get option and run ab test")
+
+    # Number of requests to perform.
+    parser.add_argument(u"-r", u"--requests", type=int,
+                        required=True, help=u"Number of requests to perform.")
+
+    # Server port number to use.
+    parser.add_argument(u"-p", u"--port", type=int, required=True,
+                        help=u"Server port number to use.")
+
+    # Number of clients being processed at the same time.
+    parser.add_argument(u"-c", u"--clients", type=int, required=True,
+                        help=u"Number of clients being processed at "
+                             u"the same time.")
+
+    # Filename to be requested from the servers.
+    parser.add_argument(u"-f", u"--files", type=str, required=True,
+                        help="Filename to be requested from the servers.")
+
+    # Server ip address.
+    parser.add_argument(u"-i", u"--ip", type=str, required=True,
+                        help=u"Server bind IP address.")
+
+    # Tg ip address.
+    parser.add_argument(u"-g", u"--tip", type=str, required=True,
+                        help=u"TG bind IP address.")
+
+    # Specify SSL/TLS cipher suite.
+    parser.add_argument(u"-z", u"--cipher", type=str, default=u"0",
+                        help=u"Specify SSL/TLS cipher.")
+
+    # Specify SSL/TLS protocol.
+    parser.add_argument(u"-t", u"--protocol", type=str, default=u"0",
+                        help=u"Specify SSL/TLS protocol.")
+
+    # Mode: RPS or CPS.
+    parser.add_argument(u"-m", u"--mode", type=str, required=True,
+                        help=u"Send requests mode:RPS/CPS.")
+
+    args = parser.parse_args()
+
+    req_num = args.requests
+    port = args.port
+    cli_num = args.clients
+    files = args.files
+    ip_address = args.ip
+    tg_address = args.tip
+    cipher = args.cipher
+    protocol = args.protocol
+    mode = args.mode
+
+    if req_num == 0:
+        print(u"Failed number of req_num!")
+        return 1
+
+    # The number of processing units available to the current process.
+    _, cpu_num = subprocess.getstatusoutput(u"nproc --all")
+    cpu_num = int(cpu_num)
+    if cpu_num > 70:
+        cpu_num = 70
+
+    # Requests and Clients are evenly distributed on each CPU.
+    per_req = round(req_num / cpu_num)
+    per_cli = round(cli_num / cpu_num)
+
+    # Revise rounding request, This will be in the first ab request
+    all_total = per_req * cpu_num
+    per_req_1st = per_req + (req_num - all_total)
+
+    results = []
+    # Start process pool.
+    pool = Pool(processes=cpu_num)
+
+    for i in range(1, cpu_num + 1):
+        results.append(
+            pool.apply_async(one, (
+                i, per_req_1st if i == 1 else per_req, per_cli, cipher,
+                protocol, ip_address, tg_address, files, port, mode)))
+
+    pool.close()
+    pool.join()
+
+    info_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+
+    # Statistical test results.
+    for res in results:
+        stats = res.get()
+        if stats:
+            info_list = [a + b for a, b in zip(info_list, stats)]
+
+    # Output results.
+    print(f"Transfer Rate: {round(info_list[6], 2)} [Kbytes/sec]")
+    print(f"Latency: {round(info_list[4] / 8, 2)} ms")
+    print(f"Connection {mode} rate:{round(info_list[3], 2)} per sec")
+    print(f"Total data transferred: {round(info_list[2])} bytes")
+    print(f"Completed requests: {round(info_list[0])} ")
+    print(f"Failed requests: {round(info_list[1])} ")
+
+
+def one(cpu, requests, clients, cipher, protocol, ip_addr, tg_addr, files, port,
+        mode):
+    """Run one test.
+
+    :param cpu: Core number id.
+    :param requests: Request number.
+    :param clients: Clients number.
+    :param cipher: Specify SSL/TLS cipher suite.
+    :param protocol: Specify SSL/TLS protocol.
+    :param ip_addr: Server ip address.
+    :param tg_addr: Tg ip address.
+    :param files: Filename to be requested from the servers.
+    :param port: Server port.
+    :type cpu: int
+    :type requests: int
+    :type clients: int
+    :type cipher: str
+    :type protocol: str
+    :type ip_addr: str
+    :type tg_addr: str
+    :type files: str
+    :type port: int
+    :type mode: str
+    :returns: Test results.
+    :rtype: list
+    """
+
+    cmd = f"sudo -E -S taskset --cpu-list {cpu} ab -n {requests} -c {clients}"
+    cmd = f"{cmd} -B {tg_addr} -r "
+    if mode == u"rps":
+        cmd = f"{cmd} -k"
+
+    if port == 80:
+        cmd = f"{cmd} http://{ip_addr}:{port}/{files}"
+    else:
+        cmd = f"{cmd} -Z {cipher} -f {protocol}"
+        cmd = f"{cmd} https://{ip_addr}:{port}/{files}"
+
+    _, output = subprocess.getstatusoutput(cmd)
+    ret = _parse_output(output)
+
+    return ret
+
+
+def _parse_output(msg):
+    """Parse the stdout with the results.
+
+    :param msg: stdout of ab.
+    :type msg: str
+    :returns: Parsed results.
+    :rtype: list
+    """
+
+    msg_lst = msg.splitlines(False)
+
+    stats = []
+    for line in msg_lst:
+        if u"Requests per second" in line:
+            stats.append(
+                _float_number(re.search(REGEX_RPS, line).group(1))
+            )
+        elif u"Time per request" in line:
+            stats.append(
+                _float_number(re.search(REGEX_LATENCY, line).group(1))
+            )
+        elif u"Transfer rate" in line:
+            stats.append(
+                _float_number(re.search(REGEX_TR, line).group(1))
+            )
+        elif u"Total transferred" in line:
+            stats.append(
+                _float_number(re.search(REGEX_TT, line).group(1))
+            )
+        elif u"Complete requests" in line:
+            stats.append(
+                _float_number(re.search(REGEX_OK_NUM, line).group(1))
+            )
+        elif u"Failed requests" in line:
+            stats.append(
+                _float_number(re.search(REGEX_FAILED_NUM, line).group(1))
+            )
+
+    return stats
+
+
+def _float_number(num):
+    """float value of the number.
+
+    :param num: Number to evaluate.
+    :type num: str
+    :returns: float number.
+    :rtype: float
+    """
+
+    val = re.search(REGEX_NUM, num)
+    try:
+        val_num = float(val.group(1))
+    except ValueError:
+        raise RuntimeError(u"The output of ab does not include the results.")
+    return val_num
+
+
+if __name__ == "__main__":
+    main()
diff --git a/resources/tools/ab/ABTools.py b/resources/tools/ab/ABTools.py
new file mode 100644
index 0000000000..cbd1adf21f
--- /dev/null
+++ b/resources/tools/ab/ABTools.py
@@ -0,0 +1,193 @@
+# Copyright (c) 2021 Intel and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""ab implementation into CSIT framework."""
+
+from robot.api import logger
+from resources.libraries.python.topology import NodeType
+from resources.libraries.python.Constants import Constants
+from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.OptionString import OptionString
+
+
+class ABTools:
+    """This class implements:
+    - Get ab command.
+    - Check ab version.
+    """
+
+    @staticmethod
+    def get_cmd_options(**kwargs):
+        """Create  parameters options.
+
+        :param kwargs: Dict of cmd parameters.
+        :type kwargs: dict
+        :returns: Cmd parameters.
+        :rtype: OptionString
+        """
+        cmd = OptionString()
+        cmd.add(u"python3")
+        dirname = f"{Constants.REMOTE_FW_DIR}/resources/tools/ab"
+        cmd.add(f"{dirname}/ABFork.py")
+        cmd_options = OptionString(prefix=u"-")
+        # Number of requests to perform.
+        cmd_options.add_with_value_from_dict(u"r", u"requests", kwargs)
+        # Server port number to use.
+        cmd_options.add_with_value_from_dict(u"p", u"port", kwargs)
+        # Number of clients being processed at the same time.
+        cmd_options.add_with_value_from_dict(u"c", u"clients", kwargs)
+        # Filename to be requested from the servers.
+        cmd_options.add_with_value_from_dict(u"f", u"files", kwargs)
+        # Server ip address.
+        cmd_options.add_with_value_from_dict(u"i", u"ip", kwargs)
+        # tg ip address.
+        cmd_options.add_with_value_from_dict(u"g", u"tip", kwargs)
+        # Specify SSL/TLS cipher suite.
+        cmd_options.add_with_value_from_dict(u"z", u"cipher", kwargs, default=0)
+        # Specify SSL/TLS protocol.
+        cmd_options.add_with_value_from_dict(u"t", u"protocol", kwargs,
+                                             default=0)
+        # Mode: RPS or CPS.
+        cmd_options.add_with_value_from_dict(u"m", u"mode", kwargs)
+        return cmd.extend(cmd_options)
+
+    @staticmethod
+    def check_ab(tg_node):
+        """Check if ab is installed on the TG node.
+
+        :param tg_node: Topology node.
+        :type tg_node: dict
+        :raises: RuntimeError if the given node is not a TG node or if the
+            command is not available.
+        """
+
+        if tg_node[u"type"] != NodeType.TG:
+            raise RuntimeError(u"Node type is not a TG!")
+
+        cmd = u"command -v ab"
+        message = u"ab not installed on TG node!"
+        exec_cmd_no_error(tg_node, cmd, message=message)
+
+    @staticmethod
+    def run_ab(tg_node, ip_addr, tg_addr, tls_tcp, cipher, files_num, rps_cps,
+               r_total, c_total, port, protocol=u"TLS1.3"):
+        """ Run ab test.
+
+        :param tg_node: Topology node.
+        :param ip_addr: Sut ip address.
+        :param tg_addr: Tg ip address.
+        :param tls_tcp: TLS or TCP.
+        :param cipher: Specify SSL/TLS cipher suite.
+        :param files_num: Filename to be requested from the servers.
+        The file is named after the file size.
+        :param rps_cps: RPS or CPS.
+        :param r_total: Requests total.
+        :param r_total: Clients total.
+        :param port: Server listen port.
+        :param protocol: TLS Protocol.
+        :type tg_node: dict
+        :type ip_addr: str
+        :type tg_addr: str
+        :type tls_tcp: str
+        :type cipher: str
+        :type files_num: int
+        :type rps_cps: str
+        :type r_total: int
+        :type c_total: int
+        :type port: int
+        :type protocol: str
+        :returns: Message with measured data.
+        :rtype: str
+        :raises: RuntimeError if node type is not a TG.
+        """
+        if files_num == 0:
+            files = u"return"
+        elif files_num >= 1024:
+            files = f"{int(files_num / 1024)}KB.json"
+        else:
+            files = f"{files_num}B.json"
+
+        cmd = ABTools.get_cmd_options(
+            requests=r_total,
+            clients=c_total,
+            ip=ip_addr,
+            tip=tg_addr,
+            files=files,
+            cipher=cipher,
+            protocol=protocol,
+            port=port,
+            mode=rps_cps,
+        )
+        stdout, _ = exec_cmd_no_error(tg_node, cmd, timeout=180, sudo=True,
+                                      message=u"ab runtime error!")
+        log_msg = ABTools._parse_ab_output(stdout, rps_cps, tls_tcp)
+
+        logger.info(log_msg)
+
+        return log_msg
+
+    @staticmethod
+    def _parse_ab_output(msg, rps_cps, tls_tcp):
+        """Parse the ab stdout with the results.
+
+        :param msg: Ab Stdout.
+        :param rps_cps: RPS or CPS.
+        :param tls_tcp: TLS or TCP.
+        :type msg: str
+        :type rps_cps: str
+        :type tls_tcp: str
+        :return: Message with measured data.
+        :rtype: str
+        """
+
+        msg_lst = msg.splitlines(keepends=False)
+
+        total_cps = u""
+        latency = u""
+        processing = u""
+        complete_req = u""
+        failed_req = u""
+        total_bytes = u""
+        rate = u""
+
+        if tls_tcp == u"tls":
+            log_msg = u"\nMeasured HTTPS values:\n"
+        else:
+            log_msg = u"\nMeasured HTTP values:\n"
+
+        for line in msg_lst:
+            if f"Connection {rps_cps} rate:" in line:
+                # rps (cps)
+                total_cps = line + u"\n"
+            elif u"Transfer Rate:" in line:
+                # Rate
+                rate = line + u"\n"
+            elif u"Latency:" in line:
+                # Latency
+                latency = line + u"\n"
+            elif u"Total data transferred" in line:
+                total_bytes = line + u"\n"
+            elif u"Completed requests" in line:
+                complete_req = line + u"\n"
+            elif u"Failed requests" in line:
+                failed_req = line + u"\n"
+
+        log_msg += rate
+        log_msg += latency
+        log_msg += processing
+        log_msg += complete_req
+        log_msg += failed_req
+        log_msg += total_bytes
+        log_msg += total_cps
+
+        return log_msg