fix: testpmd and l3fwd check state

     + print testpmd and l3fwd pid after start

Testpmd and l3fwd utility can be started but link might not be
ready yet. This fix start the utility on all DUTs and do the
check later.

Signed-off-by: Viliam Luc <vluc@cisco.com>
Change-Id: If476e22f206d9a6a0dd399879a88eafedca92bb6

7 files changed, 229 insertions, 7 deletions

diff --git a/resources/libraries/bash/entry/check_l3fwd.sh b/resources/libraries/bash/entry/check_l3fwd.sh
new file mode 100755
index 0000000000..3e53b636a3
--- /dev/null
+++ b/resources/libraries/bash/entry/check_l3fwd.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2022 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Helper functions for starting testpmd.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# FIXME: Define API contract (as opposed to just help) for bootstrap.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/dpdk.sh" || die "Source failed."
+common_dirs || die
+dpdk_l3fwd_check || die
diff --git a/resources/libraries/bash/entry/check_testpmd.sh b/resources/libraries/bash/entry/check_testpmd.sh
new file mode 100755
index 0000000000..1622b25b70
--- /dev/null
+++ b/resources/libraries/bash/entry/check_testpmd.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2022 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Helper functions for starting testpmd.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# + At least one of the following is true:
+# ++ JOB_NAME environment variable is set,
+# ++ or this entry script has access to arguments.
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# FIXME: Define API contract (as opposed to just help) for bootstrap.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/dpdk.sh" || die "Source failed."
+common_dirs || die
+dpdk_testpmd_check || die
diff --git a/resources/libraries/bash/function/dpdk.sh b/resources/libraries/bash/function/dpdk.sh
index 491f03eec0..bf1e8e337e 100644
--- a/resources/libraries/bash/function/dpdk.sh
+++ b/resources/libraries/bash/function/dpdk.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2022 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -245,6 +245,7 @@ function dpdk_l3fwd () {
     for attempt in {1..60}; do
         echo "Checking if l3fwd is alive, attempt nr ${attempt}"
         if fgrep "L3FWD: entering main loop on lcore" screenlog.0; then
+            cat screenlog.0
             exit 0
         fi
         sleep 1
@@ -255,6 +256,38 @@ function dpdk_l3fwd () {
 }
 
 
+function dpdk_l3fwd_check () {
+
+    # DPDK l3fwd check state.
+
+    set -exuo pipefail
+
+    for attempt in {1..60}; do
+        echo "Checking if l3fwd state is ok, attempt nr ${attempt}"
+        if fgrep "Port 0 Link up" screenlog.0 && \
+        fgrep "Port 1 Link up" screenlog.0; then
+            cat screenlog.0
+            dpdk_l3fwd_pid
+            exit 0
+        fi
+        sleep 1
+    done
+    cat screenlog.0
+
+    exit 1
+}
+
+
+function dpdk_l3fwd_pid () {
+    l3fwd_pid="$(pidof dpdk-l3fwd)"
+    if [ ! -z "${l3fwd_pid}" ]; then
+        echo "L3fwd process ID: ${l3fwd_pid}"
+    else
+        echo "L3fwd not running!"
+    fi
+}
+
+
 function dpdk_precheck () {
 
     # Precheck system settings (nr_hugepages, max_map_count).
@@ -299,9 +332,10 @@ function dpdk_testpmd () {
 
     for attempt in {1..60}; do
         echo "Checking if testpmd is alive, attempt nr ${attempt}"
-         if fgrep "Press enter to exit" screenlog.0; then
-             cat screenlog.0
-             exit 0
+        if fgrep "Press enter to exit" screenlog.0; then
+            cat screenlog.0
+            dpdk_testpmd_pid
+            exit 0
         fi
         sleep 1
     done
@@ -309,3 +343,34 @@ function dpdk_testpmd () {
 
     exit 1
 }
+
+
+function dpdk_testpmd_check () {
+
+    # DPDK testpmd check links state.
+
+    set -exuo pipefail
+
+    for attempt in {1..60}; do
+        echo "Checking if testpmd links state changed, attempt nr ${attempt}"
+        if fgrep "Port 0: link state change event" screenlog.0 && \
+        fgrep "Port 1: link state change event" screenlog.0; then
+            cat screenlog.0
+            exit 0
+        fi
+        sleep 1
+    done
+    cat screenlog.0
+
+    exit 1
+}
+
+
+function dpdk_testpmd_pid () {
+    testpmd_pid="$(pidof dpdk-testpmd)"
+    if [ ! -z "${testpmd_pid}" ]; then
+        echo "Testpmd process ID: ${testpmd_pid}"
+    else
+        echo "Testpmd not running!"
+    fi
+}
diff --git a/resources/libraries/python/DPDK/L3fwdCheck.py b/resources/libraries/python/DPDK/L3fwdCheck.py
new file mode 100644
index 0000000000..a2f3ce655e
--- /dev/null
+++ b/resources/libraries/python/DPDK/L3fwdCheck.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2022 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from resources.libraries.python.Constants import Constants
+from resources.libraries.python.DpdkUtil import DpdkUtil
+from resources.libraries.python.ssh import exec_cmd_no_error, exec_cmd
+from resources.libraries.python.topology import NodeType, Topology
+
+
+class L3fwdCheck:
+    """Test the DPDK l3fwd is ready."""
+
+    @staticmethod
+    def check_l3fwd(node):
+        """
+        Execute the l3fwd check on the DUT node.
+
+        :param node: DUT node.
+        :type node: dict
+        :raises RuntimeError: If the script "check_l3fwd.sh" fails.
+        """
+        if node[u"type"] == NodeType.DUT:
+            command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
+                f"/entry/check_l3fwd.sh"
+            message = f"Failed to check l3fwd state at node {node['host']}"
+            exec_cmd_no_error(node, command, timeout=1800, message=message)
diff --git a/resources/libraries/python/DPDK/TestpmdCheck.py b/resources/libraries/python/DPDK/TestpmdCheck.py
new file mode 100644
index 0000000000..fb6f4dc791
--- /dev/null
+++ b/resources/libraries/python/DPDK/TestpmdCheck.py
@@ -0,0 +1,35 @@
+# Copyright (c) 2022 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from resources.libraries.python.Constants import Constants
+from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.topology import NodeType
+
+
+class TestpmdCheck:
+    """Test the DPDK testpmd is ready."""
+
+    @staticmethod
+    def check_testpmd(node):
+        """
+        Execute the testpmd check on the DUT node.
+
+        :param node: DUT node.
+        :type node: dict
+        :raises RuntimeError: If the script "check_testpmd.sh" fails.
+        """
+        if node[u"type"] == NodeType.DUT:
+            command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
+                f"/entry/check_testpmd.sh"
+            message = f"Failed to check testpmd state at node {node['host']}"
+            exec_cmd_no_error(node, command, timeout=1800, message=message)
diff --git a/resources/libraries/python/DPDK/TestpmdTest.py b/resources/libraries/python/DPDK/TestpmdTest.py
index ed53acb0b5..f88b368e93 100644
--- a/resources/libraries/python/DPDK/TestpmdTest.py
+++ b/resources/libraries/python/DPDK/TestpmdTest.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2022 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -74,7 +74,7 @@ class TestpmdTest:
                 pmd_rxq=queue_nums,
                 pmd_txq=queue_nums,
                 pmd_nb_cores=nb_cores,
-                pmd_disable_link_check=True,
+                pmd_disable_link_check=False,
                 pmd_auto_start=True,
                 pmd_numa=True
             )
diff --git a/resources/libraries/robot/dpdk/default.robot b/resources/libraries/robot/dpdk/default.robot
index 9fd6379afd..b63f245d07 100644
--- a/resources/libraries/robot/dpdk/default.robot
+++ b/resources/libraries/robot/dpdk/default.robot
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2022 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -15,7 +15,9 @@
 | Library | resources.libraries.python.InterfaceUtil
 | Library | resources.libraries.python.CpuUtils
 | Library | resources.libraries.python.DPDK.TestpmdTest
+| Library | resources.libraries.python.DPDK.TestpmdCheck
 | Library | resources.libraries.python.DPDK.L3fwdTest
+| Library | resources.libraries.python.DPDK.L3fwdCheck
 | Library | Collections
 
 *** Keywords ***
@@ -50,6 +52,9 @@
 | | | ... | ${nodes['${dut}']} | ${${dut}_pf1}[0] | ${${dut}_pf2}[0]
 | | | ... | ${cpu_dp} | ${dp_count_int} | ${rxq_count_int} | ${jumbo_frames}
 | | | ... | ${nic_rxq_size} | ${nic_txq_size}
+| | FOR | ${dut} | IN | @{duts}
+| | | Check testpmd
+| | | ... | ${nodes['${dut}']}
 | | END
 
 | Start l3fwd on all DUTs
@@ -82,4 +87,7 @@
 | | | Start l3fwd
 | | | ... | ${nodes} | ${nodes['${dut}']} | ${${dut}_pf1}[0] | ${${dut}_pf2}[0]
 | | | ... | ${cpu_dp} | ${dp_count_int} | ${rxq_count_int} | ${jumbo_frames}
+| | FOR | ${dut} | IN | @{duts}
+| | | Check l3fwd
+| | | ... | ${nodes['${dut}']}
 | | END