Performance: DPDK refactor

+ Rework BASH scripts (more code to python).
+ Move BASH into libraries.
+ Allows RDMA usage.
+ Fix 9000B tests.
+ Rename confusing l2fwd -> testpmd.
+ Fix suite setup.
+ Fix PCI whitelist to not accidentally pickup wrong interface.
+ Fix deprecated DPDK cli arguments.

- MLX5 jumbo are disabled on NIC (i will increase separately).
  https://jenkins.fd.io/job/csit-dpdk-perf-verify-master-2n-clx/6/console (l3fwd still broken)
- MLX5 IMIX seems to be some TRex issue with IMIX for mlx5 (i will handle separately)

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: I31d1b67305fa247cb5e1f57e739d3ef30dc1a04b

19 files changed, 1062 insertions, 240 deletions

diff --git a/resources/libraries/bash/entry/cleanup_dpdk.sh b/resources/libraries/bash/entry/cleanup_dpdk.sh
new file mode 100755
index 0000000000..726f98c1a1
--- /dev/null
+++ b/resources/libraries/bash/entry/cleanup_dpdk.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Cleanup the DPDK framework on the DUT node. Bind interfaces to driver.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# FIXME: Define API contract (as opposed to just help) for bootstrap.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/dpdk.sh" || die "Source failed."
+common_dirs || die
+dpdk_kill || die
+dpdk_bind "${@}" || die
diff --git a/resources/libraries/bash/entry/init_dpdk.sh b/resources/libraries/bash/entry/init_dpdk.sh
new file mode 100755
index 0000000000..910ade38f9
--- /dev/null
+++ b/resources/libraries/bash/entry/init_dpdk.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Initialize the DPDK framework on the DUT node. Bind interfaces to driver.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# FIXME: Define API contract (as opposed to just help) for bootstrap.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/dpdk.sh" || die "Source failed."
+common_dirs || die
+dpdk_bind "${@}" || die
diff --git a/resources/libraries/bash/entry/install_dpdk.sh b/resources/libraries/bash/entry/install_dpdk.sh
new file mode 100755
index 0000000000..d87aa5a832
--- /dev/null
+++ b/resources/libraries/bash/entry/install_dpdk.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install the DPDK framework on the DUT node. Check prerequisites.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# FIXME: Define API contract (as opposed to just help) for bootstrap.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/dpdk.sh" || die "Source failed."
+common_dirs || die
+dpdk_extract || die
+dpdk_compile || die
+dpdk_precheck || die
diff --git a/resources/libraries/bash/entry/patch_l3fwd.sh b/resources/libraries/bash/entry/patch_l3fwd.sh
new file mode 100755
index 0000000000..0ce02a0047
--- /dev/null
+++ b/resources/libraries/bash/entry/patch_l3fwd.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Recompile DPDK l3fwd sample app with patching.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# FIXME: Define API contract (as opposed to just help) for bootstrap.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/dpdk.sh" || die "Source failed."
+common_dirs || die
+dpdk_l3fwd_compile "${@}" || die
diff --git a/resources/libraries/bash/entry/patch_l3fwd_flip_routes b/resources/libraries/bash/entry/patch_l3fwd_flip_routes
new file mode 100755
index 0000000000..e0b204eb9c
--- /dev/null
+++ b/resources/libraries/bash/entry/patch_l3fwd_flip_routes
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+patch --ignore-whitespace l3fwd_lpm.c <<"_EOF"
+44,45c44,45
+<       {IPv4(1, 1, 1, 0), 24, 0},
+<       {IPv4(2, 1, 1, 0), 24, 1},
+---
+>       {IPv4(1, 1, 1, 0), 24, 1},
+>       {IPv4(2, 1, 1, 0), 24, 0},
+_EOF
diff --git a/resources/libraries/bash/entry/run_l3fwd.sh b/resources/libraries/bash/entry/run_l3fwd.sh
new file mode 100755
index 0000000000..5716dec59a
--- /dev/null
+++ b/resources/libraries/bash/entry/run_l3fwd.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Helper functions for starting l3fwd.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# FIXME: Define API contract (as opposed to just help) for bootstrap.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/dpdk.sh" || die "Source failed."
+common_dirs || die
+dpdk_kill || die
+dpdk_l3fwd "${@}" || die
diff --git a/resources/libraries/bash/entry/run_testpmd.sh b/resources/libraries/bash/entry/run_testpmd.sh
new file mode 100755
index 0000000000..6208487774
--- /dev/null
+++ b/resources/libraries/bash/entry/run_testpmd.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Helper functions for starting testpmd.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# FIXME: Define API contract (as opposed to just help) for bootstrap.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/dpdk.sh" || die "Source failed."
+common_dirs || die
+dpdk_kill || die
+dpdk_testpmd "${@}" || die
diff --git a/resources/libraries/bash/function/dpdk.sh b/resources/libraries/bash/function/dpdk.sh
new file mode 100644
index 0000000000..da46c6d49c
--- /dev/null
+++ b/resources/libraries/bash/function/dpdk.sh
@@ -0,0 +1,362 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+
+function common_dirs () {
+
+    # Set global variables, create some directories (without touching content).
+    # This function assumes running in remote testbed. It might override other
+    # functions if included from common.sh.
+
+    # Variables set:
+    # - BASH_FUNCTION_DIR - Path to existing directory this file is located in.
+    # - DPDK_DIR - Path to DPDK framework.
+    # - CSIT_DIR - Path to CSIT framework.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    this_file=$(readlink -e "${BASH_SOURCE[0]}") || {
+        die "Some error during locating of this source file."
+    }
+    BASH_FUNCTION_DIR=$(dirname "${this_file}") || {
+        die "Some error during dirname call."
+    }
+    CSIT_DIR=$(readlink -e "/tmp/openvpp-testing") || {
+        die "Readlink failed."
+    }
+    mkdir -p "${CSIT_DIR}/dpdk" || die "Mkdir failed."
+    DPDK_DIR=$(readlink -e "${CSIT_DIR}/dpdk") || {
+        die "Readlink failed."
+    }
+}
+
+
+function dpdk_bind () {
+
+    # Bind interfaces to driver.
+    #
+    # Variables read:
+    # - DPDK_DIR - Path to DPDK framework.
+    # - @ - Script cli arguments.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    pushd "${DPDK_DIR}/" || die "Pushd failed"
+
+    sudo ./usertools/dpdk-devbind.py -b "${@}" || {
+        die "Bind ${@} failed"
+    }
+
+    popd || die "Popd failed"
+}
+
+
+function dpdk_compile () {
+
+    # Compile DPDK archive.
+    #
+    # Variables read:
+    # - DPDK_DIR - Path to DPDK framework.
+    # - CSIT_DIR - Path to CSIT framework.
+    # Variables exported:
+    # - RTE_SDK - DPDK directory.
+    # - RTE_TARGET - Make targed of DPDK framework.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    arch=$(uname -m) || {
+        die "Get CPU architecture failed."
+    }
+
+    # DPDK prefers "arm64" to "aarch64" and does not allow arm64 native target.
+    if [ ${arch} == "aarch64" ]; then
+        arch="arm64"
+        machine="armv8a"
+    else
+        machine="native"
+    fi
+
+    # Patch settings.
+    sed_mlx="s/^CONFIG_RTE_LIBRTE_MLX5_PMD=n/CONFIG_RTE_LIBRTE_MLX5_PMD=y/g"
+    sed_i40e="s/^CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC=n/CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC=y/g"
+    sed_file="./config/common_base"
+
+    pushd "${DPDK_DIR}" || die "Pushd failed"
+    if ( lsmod || die ) | fgrep mlx; then
+        sed -i "${sed_mlx}" "${sed_file}" || die
+    fi
+
+    sed -i "${sed_i40e}" "${sed_file}" || die "Patch failed"
+
+    # Compile
+    make install T="${arch}"-"${machine}"-linuxapp-gcc -j || {
+        die "Failed to compile DPDK!"
+    }
+    popd || die "Popd failed"
+
+    # Compile the l3fwd.
+    export RTE_SDK="${DPDK_DIR}/"
+    export RTE_TARGET="${arch}-${machine}-linuxapp-gcc"
+    # Patch settings.
+    sed_rxd="s/^#define RTE_TEST_RX_DESC_DEFAULT 128/#define RTE_TEST_RX_DESC_DEFAULT 1024/g"
+    sed_txd="s/^#define RTE_TEST_TX_DESC_DEFAULT 512/#define RTE_TEST_TX_DESC_DEFAULT 1024/g"
+    sed_file="./main.c"
+    pushd "${RTE_SDK}"/examples/l3fwd || die "Pushd failed"
+    sed -i "${sed_rxd}" "${sed_file}" || die "Patch failed"
+    sed -i "${sed_txd}" "${sed_file}" || die "Patch failed"
+    make clean || die "Failed to compile l3fwd"
+    make -j || die "Failed to compile l3fwd"
+    popd || die "Popd failed"
+}
+
+
+function dpdk_extract () {
+
+    # Extract DPDK framework.
+    #
+    # Variables read:
+    # - DPDK_DIR - Path to DPDK framework.
+    # - CSIT_DIR - Path to CSIT framework.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    pushd "${CSIT_DIR}" || die "Pushd failed"
+    tar -xvf download_dir/dpdk*.tar.xz --strip=1 --directory "${DPDK_DIR}" || {
+        die "Failed to extract DPDK!"
+    }
+}
+
+
+function dpdk_kill () {
+
+    # Kill testpmd and/or l3fwd if running.
+
+    # Function will be noisy and requires custom error handling.
+    set -x
+    set +e
+
+    # Try to kill the testpmd.
+    sudo pgrep testpmd
+    if [ $? -eq "0" ]; then
+        success=false
+        sudo pkill testpmd
+        for attempt in {1..60}; do
+            echo "Checking if testpmd is still alive, attempt nr ${attempt}"
+            sudo pgrep testpmd
+            if [ $? -eq "1" ]; then
+                success=true
+                break
+            fi
+            echo "testpmd is still alive, waiting 1 second"
+            sleep 1
+        done
+        if [ "$success" = false ]; then
+            echo "The command sudo pkill testpmd failed"
+            sudo pkill -9 testpmd
+            exit 1
+        fi
+    else
+        echo "testpmd is not running"
+    fi
+
+    # Try to kill the l3fwd.
+    l3fwd_pid="$(pgrep l3fwd)"
+    if [ ! -z "${l3fwd_pid}" ]; then
+        success=false
+        sudo kill -15 "${l3fwd_pid}"
+        for attempt in {1..60}; do
+            echo "Checking if l3fwd is still alive, attempt nr ${attempt}"
+            l3fwd_pid="$(pgrep l3fwd)"
+            if [ -z "${l3fwd_pid}" ]; then
+                success=true
+                break
+            fi
+            echo "l3fwd is still alive, waiting 1 second"
+            sleep 1
+        done
+        if [ "${success}" = false ]; then
+            echo "The command sudo kill -15 l3fwd failed"
+            sudo kill -9 "${l3fwd_pid}"
+            exit 1
+        fi
+    else
+        echo "l3fwd is not running"
+    fi
+
+    # Remove hugepages
+    sudo rm -f /dev/hugepages/* || die "Removing hugepages failed!"
+}
+
+
+function dpdk_l3fwd_compile () {
+
+    # Compile DPDK l3fwd sample app.
+    #
+    # Variables read:
+    # - DPDK_DIR - Path to DPDK framework.
+    # - CSIT_DIR - Path to CSIT framework.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    arch=$(uname -m) || {
+        die "Get CPU architecture failed."
+    }
+
+    # DPDK prefers "arm64" to "aarch64" and does not allow arm64 native target.
+    if [ ${arch} == "aarch64" ]; then
+        arch="arm64"
+        machine="armv8a"
+    else
+        machine="native"
+    fi
+
+    # Compile the l3fwd.
+    export RTE_SDK="${DPDK_DIR}/"
+    export RTE_TARGET="${arch}-${machine}-linuxapp-gcc"
+    # Patch settings.
+    sed_rxd="s/^#define RTE_TEST_RX_DESC_DEFAULT 128/#define RTE_TEST_RX_DESC_DEFAULT 2048/g"
+    sed_txd="s/^#define RTE_TEST_TX_DESC_DEFAULT 512/#define RTE_TEST_TX_DESC_DEFAULT 2048/g"
+    sed_file="./main.c"
+    pushd "${RTE_SDK}"/examples/l3fwd || die "Pushd failed"
+    sed -i "${sed_rxd}" "${sed_file}" || die "Patch failed"
+    sed -i "${sed_txd}" "${sed_file}" || die "Patch failed"
+    chmod +x ${1} && source ${1} || die "Patch failed"
+    make clean || die "Failed to compile l3fwd"
+    make -j || die "Failed to compile l3fwd"
+    popd || die "Popd failed"
+}
+
+
+function dpdk_l3fwd () {
+
+    # Run DPDK l3fwd.
+    #
+    # Variables read:
+    # - DPDK_DIR - Path to DPDK framework.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    arch=$(uname -m) || {
+        die "Get CPU architecture failed."
+    }
+
+    # DPDK prefers "arm64" to "aarch64" and does not allow arm64 native target.
+    if [ ${arch} == "aarch64" ]; then
+        arch="arm64"
+        machine="armv8a"
+    else
+        machine="native"
+    fi
+
+    rm -f screenlog.0 || true
+    binary="${DPDK_DIR}/examples/l3fwd/build/app/l3fwd"
+
+    sudo sh -c "screen -dmSL DPDK-test ${binary} ${@}" || {
+        die "Failed to start l3fwd"
+    }
+
+    for attempt in {1..60}; do
+        echo "Checking if l3fwd is alive, attempt nr ${attempt}"
+        if fgrep "L3FWD: entering main loop on lcore" screenlog.0; then
+            exit 0
+        fi
+        sleep 1
+    done
+    cat screenlog.0
+
+    exit 1
+}
+
+
+function dpdk_precheck () {
+
+    # Precheck system settings (nr_hugepages, max_map_count).
+    #
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    sys_hugepage="$(< /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages)"
+    node0="/sys/devices/system/node/node0/hugepages/hugepages-2048kB/"
+    node1="/sys/devices/system/node/node1/hugepages/hugepages-2048kB/"
+    if [ ${sys_hugepage} -lt 4096 ]; then
+        echo 2048 | sudo tee "${node0}"/nr_hugepages || die
+        echo 2048 | sudo tee "${node1}"/nr_hugepages || die
+    fi
+
+    sys_map="$(< /proc/sys/vm/max_map_count)"
+    if [ ${sys_map} -lt 200000 ]; then
+        echo 200000 | sudo tee /proc/sys/vm/max_map_count || die
+    fi
+}
+
+
+function dpdk_testpmd () {
+
+    # Run DPDK testpmd.
+    #
+    # Variables read:
+    # - DPDK_DIR - Path to DPDK framework.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    arch=$(uname -m) || {
+        die "Get CPU architecture failed."
+    }
+
+    # DPDK prefers "arm64" to "aarch64" and does not allow arm64 native target.
+    if [ ${arch} == "aarch64" ]; then
+        arch="arm64"
+        machine="armv8a"
+    else
+        machine="native"
+    fi
+
+    rm -f screenlog.0 || true
+    binary="${DPDK_DIR}/${arch}-${machine}-linuxapp-gcc/app/testpmd"
+
+    sudo sh -c "screen -dmSL DPDK-test ${binary} ${@}" || {
+        die "Failed to start testpmd"
+    }
+
+    for attempt in {1..60}; do
+        echo "Checking if testpmd is alive, attempt nr ${attempt}"
+        if fgrep "Press enter to exit" screenlog.0; then
+            cat screenlog.0
+            exit 0
+        fi
+        sleep 1
+    done
+    cat screenlog.0
+
+    exit 1
+}
diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py
index 7f45efe4e8..de7a120252 100644
--- a/resources/libraries/python/Constants.py
+++ b/resources/libraries/python/Constants.py
@@ -323,6 +323,38 @@ class Constants:
         u"rdma-core": u"rdma-",
     }
 
+    # Number of virtual functions of physical nic.
+    NIC_DRIVER_TO_VFS = {
+        u"vfio-pci": u"nic_vfs}= | 0",
+        u"avf": u"nic_vfs}= | 1",
+        u"rdma-core": u"nic_vfs}= | 0",
+    }
+
+    # Not each driver is supported by each NIC.
+    DPDK_NIC_NAME_TO_DRIVER = {
+        u"Cisco-VIC-1227": [u"vfio-pci"],
+        u"Cisco-VIC-1385": [u"vfio-pci"],
+        u"Intel-X520-DA2": [u"vfio-pci"],
+        u"Intel-X553": [u"vfio-pci"],
+        u"Intel-X710": [u"vfio-pci"],
+        u"Intel-XL710": [u"vfio-pci"],
+        u"Intel-XXV710": [u"vfio-pci"],
+        u"Amazon-Nitro-50G": [u"vfio-pci"],
+        u"Mellanox-CX556A": [u"mlx5_core"],
+    }
+
+    # Tags to differentiate tests for different NIC driver.
+    DPDK_NIC_DRIVER_TO_TAG = {
+        u"vfio-pci": u"DRV_VFIO_PCI",
+        u"mlx5_core": u"DRV_MLX5_CORE",
+    }
+
+    # Suite names have to be different, add prefix.
+    DPDK_NIC_DRIVER_TO_SUITE_PREFIX = {
+        u"vfio-pci": u"",
+        u"mlx5_core": u"mlx5-",
+    }
+
     # Some identifiers constructed from suite names
     # have to be independent of NIC driver used.
     # In order to remove or reject the NIC driver part,
@@ -330,13 +362,9 @@ class Constants:
     FORBIDDEN_SUITE_PREFIX_LIST = [
         prefix for prefix in NIC_DRIVER_TO_SUITE_PREFIX.values() if prefix
     ]
-
-    # Number of virtual functions of physical nic.
-    NIC_DRIVER_TO_VFS = {
-        u"vfio-pci": u"nic_vfs}= | 0",
-        u"avf": u"nic_vfs}= | 1",
-        u"rdma-core": u"nic_vfs}= | 0",
-    }
+    FORBIDDEN_SUITE_PREFIX_LIST += [
+        prefix for prefix in DPDK_NIC_DRIVER_TO_SUITE_PREFIX.values() if prefix
+    ]
 
     # TODO CSIT-1481: Crypto HW should be read from topology file instead.
     NIC_NAME_TO_CRYPTO_HW = {
diff --git a/resources/libraries/python/DPDK/DPDKTools.py b/resources/libraries/python/DPDK/DPDKTools.py
index 868d2d886d..9bb89968d2 100644
--- a/resources/libraries/python/DPDK/DPDKTools.py
+++ b/resources/libraries/python/DPDK/DPDKTools.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -12,12 +12,12 @@
 # limitations under the License.
 
 
-"""This module implements initialization and cleanup of DPDK environment."""
+"""This module implements initialization and cleanup of DPDK framework."""
 
 from robot.api import logger
 
 from resources.libraries.python.Constants import Constants
-from resources.libraries.python.ssh import SSH, exec_cmd_no_error
+from resources.libraries.python.ssh import exec_cmd_no_error
 from resources.libraries.python.topology import NodeType, Topology
 
 
@@ -28,84 +28,68 @@ class DPDKTools:
     """
 
     @staticmethod
-    def initialize_dpdk_environment(dut_node, dut_if1, dut_if2):
+    def initialize_dpdk_framework(node, if1, if2, nic_driver):
         """
-        Initialize the DPDK test environment on the dut_node.
-        Load the module uio and igb_uio, then bind the test NIC to the igb_uio.
-
-        :param dut_node: Will init the DPDK on this node.
-        :param dut_if1: DUT interface name.
-        :param dut_if2: DUT interface name.
-        :type dut_node: dict
-        :type dut_if1: str
-        :type dut_if2: str
-        :raises RuntimeError: If it fails to bind the interfaces to igb_uio.
-        """
-        if dut_node[u"type"] == NodeType.DUT:
-            pci_address1 = Topology.get_interface_pci_addr(dut_node, dut_if1)
-            pci_address2 = Topology.get_interface_pci_addr(dut_node, dut_if2)
-
-            ssh = SSH()
-            ssh.connect(dut_node)
+        Initialize the DPDK framework on the DUT node. Bind interfaces to
+        driver.
 
-            arch = Topology.get_node_arch(dut_node)
-            cmd = f"{Constants.REMOTE_FW_DIR}/tests/dpdk/dpdk_scripts" \
-                f"/init_dpdk.sh {pci_address1} {pci_address2} {arch}"
+        :param node: DUT node.
+        :param if1: DUT first interface name.
+        :param if2: DUT second interface name.
+        :param nic_driver: Interface driver.
+        :type node: dict
+        :type if1: str
+        :type if2: str
+        :type nic_driver: str
+        :raises RuntimeError: If it fails to bind the interfaces to driver.
+        """
+        if node[u"type"] == NodeType.DUT:
+            pci_address1 = Topology.get_interface_pci_addr(node, if1)
+            pci_address2 = Topology.get_interface_pci_addr(node, if2)
 
-            ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=600)
-            if ret_code != 0:
-                raise RuntimeError(
-                    f"Failed to bind the interfaces to igb_uio at node "
-                    f"{dut_node['host']}"
-                )
+            command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
+                f"/entry/init_dpdk.sh " \
+                f"{nic_driver} {pci_address1} {pci_address2}"
+            message = u"Initialize the DPDK failed!"
+            exec_cmd_no_error(node, command, timeout=600, message=message)
 
     @staticmethod
-    def cleanup_dpdk_environment(dut_node, dut_if1, dut_if2):
+    def cleanup_dpdk_framework(node, if1, if2):
         """
-        Cleanup the DPDK test environment on the DUT node.
-        Unbind the NIC from the igb_uio and bind them to the kernel driver.
-
-        :param dut_node: Will cleanup the DPDK on this node.
-        :param dut_if1: DUT interface name.
-        :param dut_if2: DUT interface name.
-        :type dut_node: dict
-        :type dut_if1: str
-        :type dut_if2: str
+        Cleanup the DPDK framework on the DUT node. Bind interfaces to
+        default driver specified in topology.
+
+        :param node: Will cleanup the DPDK on this node.
+        :param if1: DUT first interface name.
+        :param if2: DUT second interface name.
+        :type node: dict
+        :type if1: str
+        :type if2: str
         :raises RuntimeError: If it fails to cleanup the dpdk.
         """
-        if dut_node[u"type"] == NodeType.DUT:
-            pci_address1 = Topology.get_interface_pci_addr(dut_node, dut_if1)
-            if1_driver = Topology.get_interface_driver(dut_node, dut_if1)
-            pci_address2 = Topology.get_interface_pci_addr(dut_node, dut_if2)
-            if2_driver = Topology.get_interface_driver(dut_node, dut_if2)
-
-            ssh = SSH()
-            ssh.connect(dut_node)
-
-            cmd = f"{Constants.REMOTE_FW_DIR}/tests/dpdk/dpdk_scripts" \
-                f"/cleanup_dpdk.sh {if1_driver} {pci_address1} {if2_driver} " \
-                f"{pci_address2}"
-
-            ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=1200)
-            if ret_code != 0:
-                raise RuntimeError(
-                    f"Failed to cleanup the dpdk at node {dut_node[u'host']}"
-                )
+        if node[u"type"] == NodeType.DUT:
+            pci_address1 = Topology.get_interface_pci_addr(node, if1)
+            pci_address2 = Topology.get_interface_pci_addr(node, if2)
+            # We are not supporting more than one driver yet.
+            nic_driver = Topology.get_interface_driver(node, if1)
+
+            command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
+                f"/entry/cleanup_dpdk.sh " \
+                f"{nic_driver} {pci_address1} {pci_address2}"
+            message = u"Cleanup the DPDK failed!"
+            exec_cmd_no_error(node, command, timeout=1200, message=message)
 
     @staticmethod
-    def install_dpdk_test(node):
+    def install_dpdk_framework(node):
         """
-        Prepare the DPDK test environment
+        Prepare the DPDK framework on the DUT node.
 
-        :param node: Dictionary created from topology
+        :param node: Node from topology file.
         :type node: dict
-        :returns: nothing
         :raises RuntimeError: If command returns nonzero return code.
         """
-        arch = Topology.get_node_arch(node)
-
-        command = f"{Constants.REMOTE_FW_DIR}/tests/dpdk/dpdk_scripts" \
-            f"/install_dpdk.sh {arch}"
+        command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}" \
+            f"/entry/install_dpdk.sh"
         message = u"Install the DPDK failed!"
         exec_cmd_no_error(node, command, timeout=600, message=message)
 
@@ -116,14 +100,13 @@ class DPDKTools:
         logger.info(f"DPDK Version: {stdout}")
 
     @staticmethod
-    def install_dpdk_test_on_all_duts(nodes):
+    def install_dpdk_framework_on_all_duts(nodes):
         """
-        Prepare the DPDK test environment on all DUTs.
+        Prepare the DPDK framework on all DUTs.
 
         :param nodes: Nodes from topology file.
         :type nodes: dict
-        :returns: nothing
         """
         for node in list(nodes.values()):
             if node[u"type"] == NodeType.DUT:
-                DPDKTools.install_dpdk_test(node)
+                DPDKTools.install_dpdk_framework(node)
diff --git a/resources/libraries/python/DPDK/L2fwdTest.py b/resources/libraries/python/DPDK/L2fwdTest.py
deleted file mode 100644
index 76d1dcb362..0000000000
--- a/resources/libraries/python/DPDK/L2fwdTest.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""This module implements functionality which sets L2 forwarding for DPDK on
-DUT nodes.
-"""
-
-from resources.libraries.python.Constants import Constants
-from resources.libraries.python.ssh import exec_cmd_no_error
-from resources.libraries.python.topology import NodeType
-
-
-class L2fwdTest:
-    """Setup the DPDK for l2fwd performance test."""
-
-    @staticmethod
-    def start_the_l2fwd_test(
-            node, cpu_cores, nb_cores, queue_nums, jumbo_frames,
-            rxq_size=1024, txq_size=1024):
-        """
-        Execute the l2fwd on the DUT node.
-
-        :param node: Will execute the l2fwd on this node.
-        :param cpu_cores: The DPDK run cores.
-        :param nb_cores: The cores number for the forwarding.
-        :param queue_nums: The queues number for the NIC.
-        :param jumbo_frames: Indication if the jumbo frames are used (True) or
-            not (False).
-        :param rxq_size: RXQ size. Default=1024.
-        :param txq_size: TXQ size. Default=1024.
-        :type node: dict
-        :type cpu_cores: str
-        :type nb_cores: str
-        :type queue_nums: str
-        :type jumbo_frames: bool
-        :type rxq_size: int
-        :type txq_size: int
-        :raises RuntimeError: If the script "run_l2fwd.sh" fails.
-        """
-        if node[u"type"] == NodeType.DUT:
-            jumbo = u"yes" if jumbo_frames else u"no"
-            command = f"{Constants.REMOTE_FW_DIR}/tests/dpdk/dpdk_scripts" \
-                f"/run_l2fwd.sh {cpu_cores} {nb_cores} {queue_nums} {jumbo} " \
-                f"{rxq_size} {txq_size}"
-
-            message = f"Failed to execute l2fwd test at node {node['host']}"
-
-            exec_cmd_no_error(node, command, timeout=1800, message=message)
diff --git a/resources/libraries/python/DPDK/L3fwdTest.py b/resources/libraries/python/DPDK/L3fwdTest.py
index 18becabd9b..46697cfdd5 100644
--- a/resources/libraries/python/DPDK/L3fwdTest.py
+++ b/resources/libraries/python/DPDK/L3fwdTest.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -16,7 +16,8 @@ This module exists to provide the l3fwd test for DPDK on topology nodes.
 """
 
 from resources.libraries.python.Constants import Constants
-from resources.libraries.python.ssh import SSH
+from resources.libraries.python.DpdkUtil import DpdkUtil
+from resources.libraries.python.ssh import exec_cmd_no_error
 from resources.libraries.python.topology import NodeType, Topology
 
 
@@ -24,33 +25,33 @@ class L3fwdTest:
     """Test the DPDK l3fwd performance."""
 
     @staticmethod
-    def start_the_l3fwd_test(
-            nodes_info, dut_node, dut_if1, dut_if2, nb_cores, lcores_list,
-            queue_nums, jumbo_frames):
+    def start_l3fwd(
+            nodes, node, if1, if2, lcores_list, nb_cores, queue_nums,
+            jumbo_frames):
         """
         Execute the l3fwd on the dut_node.
 
-        :param nodes_info: All the nodes info in the topology file.
-        :param dut_node: Will execute the l3fwd on this node
-        :param dut_if1: The test link interface 1.
-        :param dut_if2: The test link interface 2.
-        :param nb_cores: The cores number for the forwarding
+        :param nodes: All the nodes info in the topology file.
+        :param node: DUT node.
+        :param if1: The test link interface 1.
+        :param if2: The test link interface 2.
         :param lcores_list: The lcore list string for the l3fwd routing
+        :param nb_cores: The cores number for the forwarding
         :param queue_nums: The queues number for the NIC
         :param jumbo_frames: Indication if the jumbo frames are used (True) or
                              not (False).
-        :type nodes_info: dict
-        :type dut_node: dict
-        :type dut_if1: str
-        :type dut_if2: str
-        :type nb_cores: str
+        :type nodes: dict
+        :type node: dict
+        :type if1: str
+        :type if2: str
         :type lcores_list: str
+        :type nb_cores: str
         :type queue_nums: str
         :type jumbo_frames: bool
         """
-        if dut_node[u"type"] == NodeType.DUT:
-            adj_mac0, adj_mac1 = L3fwdTest.get_adj_mac(
-                nodes_info, dut_node, dut_if1, dut_if2
+        if node[u"type"] == NodeType.DUT:
+            adj_mac0, adj_mac1, if_pci0, if_pci1 = L3fwdTest.get_adj_mac(
+                nodes, node, if1, if2
             )
 
             list_cores = [int(item) for item in lcores_list.split(u",")]
@@ -65,79 +66,92 @@ class L3fwdTest:
                     port_config += f"({port}, {queue}, {list_cores[index]}),"
                     index += 1
 
-            ssh = SSH()
-            ssh.connect(dut_node)
+            if jumbo_frames:
+                l3fwd_args = DpdkUtil.get_l3fwd_args(
+                    eal_corelist=f"0,{lcores_list}",
+                    eal_driver=False,
+                    eal_pci_whitelist0=if_pci0,
+                    eal_pci_whitelist1=if_pci1,
+                    eal_in_memory=True,
+                    pmd_config=f"\\\"{port_config.rstrip(u',')}\\\"",
+                    pmd_eth_dest_0=f"\\\"0,{adj_mac0}\\\"",
+                    pmd_eth_dest_1=f"\\\"1,{adj_mac1}\\\"",
+                    pmd_parse_ptype=True,
+                    pmd_enable_jumbo=jumbo_frames,
+                    pmd_max_pkt_len=jumbo_frames
+                )
+            else:
+                l3fwd_args = DpdkUtil.get_l3fwd_args(
+                    eal_corelist=f"0,{lcores_list}",
+                    eal_driver=False,
+                    eal_pci_whitelist0=if_pci0,
+                    eal_pci_whitelist1=if_pci1,
+                    eal_in_memory=True,
+                    pmd_config=f"\\\"{port_config.rstrip(u',')}\\\"",
+                    pmd_eth_dest_0=f"\\\"0,{adj_mac0}\\\"",
+                    pmd_eth_dest_1=f"\\\"1,{adj_mac1}\\\"",
+                    pmd_parse_ptype=True
+                )
 
-            cmd = f"{Constants.REMOTE_FW_DIR}/tests/dpdk/dpdk_scripts" \
-                f"/run_l3fwd.sh \"{lcores_list}\" " \
-                f"\"{port_config.rstrip(u',')}\" " \
-                f"{adj_mac0} {adj_mac1} {u'yes' if jumbo_frames else u'no'}"
+            command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
+                f"/entry/run_l3fwd.sh \"{l3fwd_args} -P -L -p 0x3\""
+            message = f"Failed to execute l3fwd test at node {node['host']}"
+            exec_cmd_no_error(node, command, timeout=1800, message=message)
 
-            ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=1800)
-            if ret_code != 0:
-                raise Exception(
-                    f"Failed to execute l3fwd test at node {dut_node[u'host']}"
-                )
 
     @staticmethod
-    def get_adj_mac(nodes_info, dut_node, dut_if1, dut_if2):
+    def get_adj_mac(nodes, node, if1, if2):
         """
         Get adjacency MAC addresses of the DUT node.
 
-        :param nodes_info: All the nodes info in the topology file.
-        :param dut_node: Will execute the l3fwd on this node
-        :param dut_if1: The test link interface 1.
-        :param dut_if2: The test link interface 2.
-        :type nodes_info: dict
-        :type dut_node: dict
-        :type dut_if1: str
-        :type dut_if2: str
-        :returns: Returns MAC addresses of adjacency DUT nodes.
+        :param nodes: All the nodes info in the topology file.
+        :param node: DUT node.
+        :param if1: The test link interface 1.
+        :param if2: The test link interface 2.
+        :type nodes: dict
+        :type node: dict
+        :type if1: str
+        :type if2: str
+        :returns: Returns MAC addresses of adjacency DUT nodes and PCI
+            addresses.
         :rtype: str
         """
-        if_key0 = dut_if1
-        if_key1 = dut_if2
-        if_pci0 = Topology.get_interface_pci_addr(dut_node, if_key0)
-        if_pci1 = Topology.get_interface_pci_addr(dut_node, if_key1)
+        if_key0 = if1
+        if_key1 = if2
+        if_pci0 = Topology.get_interface_pci_addr(node, if_key0)
+        if_pci1 = Topology.get_interface_pci_addr(node, if_key1)
 
-        # detect which is the port 0
+        # Detect which is the port 0.
         if min(if_pci0, if_pci1) != if_pci0:
             if_key0, if_key1 = if_key1, if_key0
-            L3fwdTest.patch_l3fwd(dut_node, u"patch_l3fwd_flip_routes")
+            L3fwdTest.patch_l3fwd(node, u"patch_l3fwd_flip_routes")
 
         adj_node0, adj_if_key0 = Topology.get_adjacent_node_and_interface(
-            nodes_info, dut_node, if_key0
+            nodes, node, if_key0
         )
         adj_node1, adj_if_key1 = Topology.get_adjacent_node_and_interface(
-            nodes_info, dut_node, if_key1
+            nodes, node, if_key1
         )
-
+        if_pci0 = Topology.get_interface_pci_addr(node, if_key0)
+        if_pci1 = Topology.get_interface_pci_addr(node, if_key1)
         adj_mac0 = Topology.get_interface_mac(adj_node0, adj_if_key0)
         adj_mac1 = Topology.get_interface_mac(adj_node1, adj_if_key1)
 
-        return adj_mac0, adj_mac1
+        return adj_mac0, adj_mac1, if_pci0, if_pci1
 
     @staticmethod
     def patch_l3fwd(node, patch):
         """
         Patch l3fwd application and recompile.
 
-        :param node: Dictionary created from topology.
+        :param node: DUT node.
         :param patch: Patch to apply.
         :type node: dict
         :type patch: str
         :raises RuntimeError: Patching of l3fwd failed.
         """
-        arch = Topology.get_node_arch(node)
-
-        ssh = SSH()
-        ssh.connect(node)
-
-        ret_code, _, _ = ssh.exec_command(
-            f"{Constants.REMOTE_FW_DIR}/tests/dpdk/dpdk_scripts/patch_l3fwd.sh "
-            f"{arch} {Constants.REMOTE_FW_DIR}/tests/dpdk/dpdk_scripts/{patch}",
-            timeout=600
-        )
-
-        if ret_code != 0:
-            raise RuntimeError(u"Patch of l3fwd failed.")
+        command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
+            f"/entry/patch_l3fwd.sh " \
+            f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}/{patch}"
+        message = f"Failed to patch l3fwd at node {node['host']}"
+        exec_cmd_no_error(node, command, timeout=1800, message=message)
diff --git a/resources/libraries/python/DPDK/TestpmdTest.py b/resources/libraries/python/DPDK/TestpmdTest.py
new file mode 100644
index 0000000000..1eccd7addf
--- /dev/null
+++ b/resources/libraries/python/DPDK/TestpmdTest.py
@@ -0,0 +1,85 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module implements functionality which sets L2 forwarding for DPDK on
+DUT nodes.
+"""
+
+from resources.libraries.python.Constants import Constants
+from resources.libraries.python.DpdkUtil import DpdkUtil
+from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.topology import NodeType, Topology
+
+
+class TestpmdTest:
+    """Setup the DPDK for testpmd performance test."""
+
+    @staticmethod
+    def start_testpmd(
+            node, if1, if2, lcores_list, nb_cores, queue_nums,
+            jumbo_frames, rxq_size=1024, txq_size=1024):
+        """
+        Execute the testpmd on the DUT node.
+
+        :param node: DUT node.
+        :param if1: The test link interface 1.
+        :param if2: The test link interface 2.
+        :param lcores_list: The DPDK run cores.
+        :param nb_cores: The cores number for the forwarding.
+        :param queue_nums: The queues number for the NIC.
+        :param jumbo_frames: Indication if the jumbo frames are used (True) or
+            not (False).
+        :param rxq_size: RXQ size. Default=1024.
+        :param txq_size: TXQ size. Default=1024.
+        :type node: dict
+        :type if1: str
+        :type if2: str
+        :type lcores_list: str
+        :type nb_cores: str
+        :type queue_nums: str
+        :type jumbo_frames: bool
+        :type rxq_size: int
+        :type txq_size: int
+        :raises RuntimeError: If the script "run_testpmd.sh" fails.
+        """
+        if node[u"type"] == NodeType.DUT:
+            if_pci0 = Topology.get_interface_pci_addr(node, if1)
+            if_pci1 = Topology.get_interface_pci_addr(node, if2)
+
+            pmd_max_pkt_len = u"9200" if jumbo_frames else u"1518"
+            testpmd_args = DpdkUtil.get_testpmd_args(
+                eal_corelist=f"0,{lcores_list}",
+                eal_driver=False,
+                eal_pci_whitelist0=if_pci0,
+                eal_pci_whitelist1=if_pci1,
+                eal_in_memory=True,
+                pmd_num_mbufs=16384,
+                pmd_fwd_mode=u"io",
+                pmd_nb_ports=u"2",
+                pmd_portmask=u"0x3",
+                pmd_max_pkt_len=pmd_max_pkt_len,
+                pmd_mbuf_size=u"16384",
+                pmd_rxd=rxq_size,
+                pmd_txd=txq_size,
+                pmd_rxq=queue_nums,
+                pmd_txq=queue_nums,
+                pmd_nb_cores=nb_cores,
+                pmd_disable_link_check=True,
+                pmd_auto_start=True,
+                pmd_numa=True
+            )
+
+            command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
+                f"/entry/run_testpmd.sh \"{testpmd_args}\""
+            message = f"Failed to execute testpmd at node {node['host']}"
+            exec_cmd_no_error(node, command, timeout=1800, message=message)
diff --git a/resources/libraries/python/DpdkUtil.py b/resources/libraries/python/DpdkUtil.py
index 3a04cbd021..24c4d57652 100644
--- a/resources/libraries/python/DpdkUtil.py
+++ b/resources/libraries/python/DpdkUtil.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -32,9 +32,20 @@ class DpdkUtil:
         options = OptionString(prefix=u"-")
         options.add(u"v")
         # Set the hexadecimal bitmask of the cores to run on.
-        options.add_with_value_from_dict(u"l", u"eal_corelist", kwargs)
+        options.add_with_value_from_dict(
+            u"l", u"eal_corelist", kwargs
+        )
+        # Add a PCI device in white list.
+        options.add_with_value_from_dict(
+            u"w", u"eal_pci_whitelist0", kwargs
+        )
+        options.add_with_value_from_dict(
+            u"w", u"eal_pci_whitelist1", kwargs
+        )
         # Set master core.
-        options.add_with_value(u"-master-lcore", u"0")
+        options.add_with_value(
+            u"-master-lcore", u"0"
+        )
         # Load an external driver. Multiple -d options are allowed.
         options.add_with_value_if_from_dict(
             u"d", u"/usr/lib/librte_pmd_virtio.so", u"eal_driver", kwargs, True
@@ -45,8 +56,8 @@ class DpdkUtil:
         return options
 
     @staticmethod
-    def get_pmd_options(**kwargs):
-        """Create PMD parameters options (without --).
+    def get_testpmd_pmd_options(**kwargs):
+        """Create PMD parameters options for testpmd (without --).
 
         :param kwargs: List of testpmd parameters.
         :type kwargs: dict
@@ -60,36 +71,77 @@ class DpdkUtil:
             u"forward-mode", u"pmd_fwd_mode", kwargs, u"io"
         )
         # Set the number of packets per burst to N.
-        options.add_equals(u"burst", 64)
+        options.add_equals(
+            u"burst", 64
+        )
         # Set the number of descriptors in the TX rings to N.
-        options.add_equals_from_dict(u"txd", u"pmd_txd", kwargs, 1024)
+        options.add_equals_from_dict(
+            u"txd", u"pmd_txd", kwargs, 1024
+        )
         # Set the number of descriptors in the RX rings to N.
-        options.add_equals_from_dict(u"rxd", u"pmd_rxd", kwargs, 1024)
+        options.add_equals_from_dict(
+            u"rxd", u"pmd_rxd", kwargs, 1024
+        )
         # Set the number of queues in the TX to N.
-        options.add_equals_from_dict(u"txq", u"pmd_txq", kwargs, 1)
+        options.add_equals_from_dict(
+            u"txq", u"pmd_txq", kwargs, 1
+        )
         # Set the number of queues in the RX to N.
-        options.add_equals_from_dict(u"rxq", u"pmd_rxq", kwargs, 1)
+        options.add_equals_from_dict(
+            u"rxq", u"pmd_rxq", kwargs, 1
+        )
         # Set the hexadecimal bitmask of offloads.
-        options.add_equals_from_dict(u"tx-offloads", u"pmd_tx_offloads", kwargs)
+        options.add_equals_from_dict(
+            u"tx-offloads", u"pmd_tx_offloads", kwargs, u"0x0"
+        )
+        # Enables numa aware allocation of mbufs.
+        options.add_if_from_dict(
+            u"numa", u"pmd_numa", kwargs, True
+        )
+        # Run by default.
+        options.add_if_from_dict(
+            u"auto-start", u"pmd_auto_start", kwargs, True
+        )
         # Set the number of mbufs to be allocated in the mbuf pools.
         options.add_equals_from_dict(
             u"total-num-mbufs", u"pmd_num_mbufs", kwargs
         )
-        # Disable hardware VLAN.
+        # Set the number of forwarding ports.
+        options.add_equals_from_dict(
+            u"nb-ports", u"pmd_nb_ports", kwargs
+        )
+        # Set the hexadecimal bitmask of the ports used by the packet
+        # forwarding test.
+        options.add_equals_from_dict(
+            u"portmask", u"pmd_portmask", kwargs
+        )
+        # Disable link status check.
         options.add_if_from_dict(
-            u"disable-hw-vlan", u"pmd_disable_hw_vlan", kwargs, True
+            u"disable-link-check", u"pmd_disable_link_check", kwargs, True
         )
         # Set the MAC address XX:XX:XX:XX:XX:XX of the peer port N
-        options.add_equals_from_dict(u"eth-peer", u"pmd_eth_peer_0", kwargs)
-        options.add_equals_from_dict(u"eth-peer", u"pmd_eth_peer_1", kwargs)
+        options.add_equals_from_dict(
+            u"eth-peer", u"pmd_eth_peer_0", kwargs
+        )
+        options.add_equals_from_dict(
+            u"eth-peer", u"pmd_eth_peer_1", kwargs
+        )
         # Set the max packet length.
-        options.add_equals_from_dict(u"max-pkt-len", u"pmd_max_pkt_len", kwargs)
+        options.add_equals_from_dict(
+            u"max-pkt-len", u"pmd_max_pkt_len", kwargs
+        )
+        # Set the max packet length.
+        options.add_equals_from_dict(
+            u"mbuf-size", u"pmd_mbuf_size", kwargs
+        )
         # Set the number of forwarding cores based on coremask.
-        options.add_equals_from_dict(u"nb-cores", u"pmd_nb_cores", kwargs)
+        options.add_equals_from_dict(
+            u"nb-cores", u"pmd_nb_cores", kwargs
+        )
         return options
 
     @staticmethod
-    def get_testpmd_cmdline(**kwargs):
+    def get_testpmd_args(**kwargs):
         """Get DPDK testpmd command line arguments.
 
         :param kwargs: Key-value testpmd parameters.
@@ -98,10 +150,25 @@ class DpdkUtil:
         :rtype: OptionString
         """
         options = OptionString()
+        options.extend(DpdkUtil.get_eal_options(**kwargs))
+        options.add(u"--")
+        options.extend(DpdkUtil.get_testpmd_pmd_options(**kwargs))
+        return options
+
+    @staticmethod
+    def get_testpmd_cmdline(**kwargs):
+        """Get DPDK testpmd command line arguments with testpmd command.
+
+        :param kwargs: Key-value testpmd parameters.
+        :type kwargs: dict
+        :returns: Command line string.
+        :rtype: OptionString
+        """
+        options = OptionString()
         options.add(u"testpmd")
         options.extend(DpdkUtil.get_eal_options(**kwargs))
         options.add(u"--")
-        options.extend(DpdkUtil.get_pmd_options(**kwargs))
+        options.extend(DpdkUtil.get_testpmd_pmd_options(**kwargs))
         return options
 
     @staticmethod
@@ -117,7 +184,7 @@ class DpdkUtil:
         cmd_options.add(u"/start-testpmd.sh")
         cmd_options.extend(DpdkUtil.get_eal_options(**kwargs))
         cmd_options.add(u"--")
-        cmd_options.extend(DpdkUtil.get_pmd_options(**kwargs))
+        cmd_options.extend(DpdkUtil.get_testpmd_pmd_options(**kwargs))
         exec_cmd_no_error(node, cmd_options, sudo=True, disconnect=True)
 
     @staticmethod
@@ -130,3 +197,53 @@ class DpdkUtil:
         """
         cmd = u"/stop-testpmd.sh"  # Completed string, simple one.
         exec_cmd_no_error(node, cmd, sudo=True, disconnect=True)
+
+    @staticmethod
+    def get_l3fwd_pmd_options(**kwargs):
+        """Create PMD parameters options for l3fwd (without --).
+
+        :param kwargs: List of l3fwd parameters.
+        :type kwargs: dict
+        :returns: PMD parameters.
+        :rtype: OptionString
+        """
+        options = OptionString(prefix=u"--")
+        # Set to use software to analyze packet type.
+        options.add_if_from_dict(
+            u"parse-ptype", u"pmd_parse_ptype", kwargs, True
+        )
+        # Set the MAC address XX:XX:XX:XX:XX:XX of the peer port N.
+        options.add_equals_from_dict(
+            u"eth-dest", u"pmd_eth_dest_0", kwargs
+        )
+        options.add_equals_from_dict(
+            u"eth-dest", u"pmd_eth_dest_1", kwargs
+        )
+        # Determines which queues from which ports are mapped to which cores.
+        options.add_equals_from_dict(
+            u"config", u"pmd_config", kwargs
+        )
+        # Enables jumbo frames.
+        options.add_if_from_dict(
+            u"enable-jumbo", u"pmd_enable_jumbo", kwargs, False
+        )
+        # Set the max packet length.
+        options.add_with_value_if_from_dict(
+            u"max-pkt-len", u"9200", u"pmd_max_pkt_len", kwargs, False
+        )
+        return options
+
+    @staticmethod
+    def get_l3fwd_args(**kwargs):
+        """Get DPDK l3fwd command line arguments.
+
+        :param kwargs: Key-value l3fwd parameters.
+        :type kwargs: dict
+        :returns: Command line string.
+        :rtype: OptionString
+        """
+        options = OptionString()
+        options.extend(DpdkUtil.get_eal_options(**kwargs))
+        options.add(u"--")
+        options.extend(DpdkUtil.get_l3fwd_pmd_options(**kwargs))
+        return options
diff --git a/resources/libraries/python/QemuUtils.py b/resources/libraries/python/QemuUtils.py
index 0113dc9e03..fb707d216b 100644
--- a/resources/libraries/python/QemuUtils.py
+++ b/resources/libraries/python/QemuUtils.py
@@ -254,15 +254,22 @@ class QemuUtils:
         :param kwargs: Key-value pairs to construct command line parameters.
         :type kwargs: dict
         """
+        pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
             eal_driver=False,
+            eal_pci_whitelist0=u"0000:00:06.0",
+            eal_pci_whitelist1=u"0000:00:07.0",
             eal_in_memory=True,
             pmd_num_mbufs=16384,
+            pmd_fwd_mode=u"io",
+            pmd_nb_ports=u"2",
+            pmd_portmask=u"0x3",
+            pmd_max_pkt_len=pmd_max_pkt_len,
+            pmd_mbuf_size=u"16384",
             pmd_rxq=kwargs[u"queues"],
             pmd_txq=kwargs[u"queues"],
             pmd_tx_offloads='0x0',
-            pmd_disable_hw_vlan=False,
             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
         )
 
@@ -274,18 +281,24 @@ class QemuUtils:
         :param kwargs: Key-value pairs to construct command line parameters.
         :type kwargs: dict
         """
+        pmd_max_pkt_len = u"9200" if kwargs[u"jumbo_frames"] else u"1518"
         testpmd_cmd = DpdkUtil.get_testpmd_cmdline(
             eal_corelist=f"0-{self._opt.get(u'smp') - 1}",
             eal_driver=False,
+            eal_pci_whitelist0=u"0000:00:06.0",
+            eal_pci_whitelist1=u"0000:00:07.0",
             eal_in_memory=True,
             pmd_num_mbufs=16384,
             pmd_fwd_mode=u"mac",
+            pmd_nb_ports=u"2",
+            pmd_portmask=u"0x3",
+            pmd_max_pkt_len=pmd_max_pkt_len,
+            pmd_mbuf_size=u"16384",
             pmd_eth_peer_0=f"0,{kwargs[u'vif1_mac']}",
             pmd_eth_peer_1=f"1,{kwargs[u'vif2_mac']}",
             pmd_rxq=kwargs[u"queues"],
             pmd_txq=kwargs[u"queues"],
             pmd_tx_offloads=u"0x0",
-            pmd_disable_hw_vlan=False,
             pmd_nb_cores=str(self._opt.get(u"smp") - 1)
         )
 
diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
index c6038588e6..3ef589d351 100644
--- a/resources/libraries/python/autogen/Regenerator.py
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -245,12 +245,41 @@ def write_default_files(in_filename, in_prolog, kwargs_list):
                 tmp2_filename
             )
             if u"DPDK" in in_prolog:
-                check_suite_tag(old_suite_tag, tmp2_prolog)
-                with open(tmp2_filename, u"wt") as file_out:
-                    file_out.write(tmp2_prolog)
-                    add_default_testcases(
-                        testcase, iface, old_suite_id, file_out, kwargs_list
+                for driver in Constants.DPDK_NIC_NAME_TO_DRIVER[nic_name]:
+                    out_filename = replace_defensively(
+                        tmp2_filename, old_suite_id,
+                        Constants.DPDK_NIC_DRIVER_TO_SUITE_PREFIX[driver] \
+                            + old_suite_id,
+                        1, u"Error adding driver prefix.", in_filename
+                    )
+                    out_prolog = replace_defensively(
+                        tmp2_prolog, u"vfio-pci", driver, 1,
+                        u"Driver name should appear once.", in_filename
+                    )
+                    out_prolog = replace_defensively(
+                        out_prolog,
+                        Constants.DPDK_NIC_DRIVER_TO_TAG[u"vfio-pci"],
+                        Constants.DPDK_NIC_DRIVER_TO_TAG[driver], 1,
+                        u"Driver tag should appear once.", in_filename
+                    )
+                    iface, suite_id, suite_tag = get_iface_and_suite_ids(
+                        out_filename
+                    )
+                    # The next replace is probably a noop, but it is safer to
+                    # maintain the same structure as for other edits.
+                    out_prolog = replace_defensively(
+                        out_prolog, old_suite_tag, suite_tag, 1,
+                        f"Perf suite tag {old_suite_tag} should appear once.",
+                        in_filename
                     )
+                    check_suite_tag(suite_tag, out_prolog)
+                    # TODO: Reorder loops so suite_id is finalized sooner.
+                    testcase = Testcase.default(suite_id)
+                    with open(out_filename, u"wt") as file_out:
+                        file_out.write(out_prolog)
+                        add_default_testcases(
+                            testcase, iface, suite_id, file_out, kwargs_list
+                        )
                 continue
             for driver in Constants.NIC_NAME_TO_DRIVER[nic_name]:
                 out_filename = replace_defensively(
diff --git a/resources/libraries/robot/dpdk/default.robot b/resources/libraries/robot/dpdk/default.robot
index 021f73d8c3..15b61e7f04 100644
--- a/resources/libraries/robot/dpdk/default.robot
+++ b/resources/libraries/robot/dpdk/default.robot
@@ -14,13 +14,13 @@
 *** Settings ***
 | Library | resources.libraries.python.InterfaceUtil
 | Library | resources.libraries.python.CpuUtils
-| Library | resources.libraries.python.DPDK.L2fwdTest
+| Library | resources.libraries.python.DPDK.TestpmdTest
 | Library | resources.libraries.python.DPDK.L3fwdTest
 | Library | Collections
 
 *** Keywords ***
-| Start L2FWD on all DUTs
-| | [Documentation] | Start the l2fwd with M worker threads and rxqueues N and
+| Start testpmd on all DUTs
+| | [Documentation] | Start the testpmd with M worker threads and rxqueues N and
 | | ... | jumbo support frames on/off on all DUTs.
 | |
 | | ... | *Arguments:*
@@ -30,7 +30,7 @@
 | |
 | | ... | *Example:*
 | |
-| | ... | \| Start L2FWD on all DUTs \| ${1} \| ${1} \| ${False} \|
+| | ... | \| Start testpmd on all DUTs \| ${1} \| ${1} \| ${False} \|
 | |
 | | [Arguments] | ${phy_cores} | ${rx_queues}=${None} | ${jumbo_frames}=${False}
 | |
@@ -52,15 +52,16 @@
 | | | ${rxq_count_int}= | Run keyword if | ${rxq_count_int} == 0
 | | | ... | Set variable | ${1}
 | | | ... | ELSE | Set variable | ${rxq_count_int}
-| | | Start the l2fwd test | ${nodes['${dut}']} | ${cpus} | ${thr_count_int}
-| | | ... | ${rxq_count_int} | ${jumbo_frames}
+| | | Start testpmd
+| | | ... | ${nodes['${dut}']} | ${${dut}_pf1}[0] | ${${dut}_pf2}[0]
+| | | ... | ${cpus} | ${thr_count_int} | ${rxq_count_int} | ${jumbo_frames}
 | | | ... | ${nic_rxq_size} | ${nic_txq_size}
 | | | Run keyword if | ${thr_count_int} > 1
 | | | ... | Set Tags | MTHREAD | ELSE | Set Tags | STHREAD
 | | | Set Tags | ${thr_count_int}T${cpu_count_int}C
 | | END
 
-| Start L3FWD on all DUTs
+| Start l3fwd on all DUTs
 | | [Documentation] | Start the l3fwd with M worker threads and rxqueues N and
 | | ... | jumbo support frames on/off on all DUTs.
 | |
@@ -71,7 +72,7 @@
 | |
 | | ... | *Example:*
 | |
-| | ... | \| Start L3FWD on all DUTs \| ${1} \| ${1} \| ${False} \|
+| | ... | \| Start l3fwd on all DUTs \| ${1} \| ${1} \| ${False} \|
 | |
 | | [Arguments] | ${phy_cores} | ${rx_queues}=${None} | ${jumbo_frames}=${False}
 | |
@@ -93,10 +94,9 @@
 | | | ${rxq_count_int}= | Run keyword if | ${rxq_count_int} == 0
 | | | ... | Set variable | ${1}
 | | | ... | ELSE | Set variable | ${rxq_count_int}
-| | | Start the l3fwd test
-| | | ... | ${nodes} | ${nodes['${dut}']}
-| | | ... | ${${dut}_pf1}[0] | ${${dut}_pf2}[0]
-| | | ... | ${thr_count_int} | ${cpus} | ${rxq_count_int} | ${jumbo_frames}
+| | | Start l3fwd
+| | | ... | ${nodes} | ${nodes['${dut}']} | ${${dut}_pf1}[0] | ${${dut}_pf2}[0]
+| | | ... | ${cpus} | ${thr_count_int} | ${rxq_count_int} | ${jumbo_frames}
 | | | Run keyword if | ${thr_count_int} > 1
 | | | ... | Set Tags | MTHREAD | ELSE | Set Tags | STHREAD
 | | | Set Tags | ${thr_count_int}T${cpu_count_int}C
diff --git a/resources/libraries/robot/shared/suite_setup.robot b/resources/libraries/robot/shared/suite_setup.robot
index 5f8c3c5b2f..0b1c0caa29 100644
--- a/resources/libraries/robot/shared/suite_setup.robot
+++ b/resources/libraries/robot/shared/suite_setup.robot
@@ -152,8 +152,8 @@
 | | ... | Additional Setup for suites which uses dpdk.
 | |
 | | FOR | ${dut} | IN | @{duts}
-| | | Initialize DPDK Environment | ${nodes['${dut}']}
-| | | ... | ${${dut}_${int}1}[0] | ${${dut}_${int}2}[0]
+| | | Initialize DPDK Framework | ${nodes['${dut}']}
+| | | ... | ${${dut}_${int}1}[0] | ${${dut}_${int}2}[0] | ${nic_driver}
 | | END
 
 | Additional Suite Setup Action For performance vf
diff --git a/resources/libraries/robot/shared/suite_teardown.robot b/resources/libraries/robot/shared/suite_teardown.robot
index 38683658d7..f76fddc9d6 100644
--- a/resources/libraries/robot/shared/suite_teardown.robot
+++ b/resources/libraries/robot/shared/suite_teardown.robot
@@ -46,7 +46,7 @@
 | | ... | Additional teardown for suites which uses dpdk.
 | |
 | | FOR | ${dut} | IN | @{duts}
-| | | Cleanup DPDK Environment
+| | | Cleanup DPDK Framework
 | | | ... | ${nodes['${dut}']} | ${${dut}_${int}1}[0] | ${${dut}_${int}2}[0]
 | | END