PAPI: Add verification of the response

Change-Id: I86afcaeae865f0af076b8dd974386a83de07bf44
Signed-off-by: Tibor Frank <tifrank@cisco.com>

1 files changed, 137 insertions, 32 deletions

diff --git a/resources/libraries/python/PapiExecutor.py b/resources/libraries/python/PapiExecutor.py
index 1971e62d95..b0ddccfbde 100644
--- a/resources/libraries/python/PapiExecutor.py
+++ b/resources/libraries/python/PapiExecutor.py
@@ -11,7 +11,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Python API executor library."""
+"""Python API executor library.
+
+This version supports only simple request / reply VPP API methods.
+
+TODO:
+ - Implement:
+   - Dump functions
+   - vpp-stats
+
+"""
 
 import binascii
 import json
@@ -22,6 +31,7 @@ from resources.libraries.python.Constants import Constants
 from resources.libraries.python.ssh import SSH, SSHTimeout
 from resources.libraries.python.PapiHistory import PapiHistory
 
+
 __all__ = ["PapiExecutor", "PapiResponse"]
 
 
@@ -30,45 +40,60 @@ class PapiResponse(object):
     code.
     """
 
-    def __init__(self, papi_reply=None, stdout="", stderr="", ret_code=None):
+    def __init__(self, papi_reply=None, stdout="", stderr="", ret_code=None,
+                 requests=None):
         """Construct the Papi response by setting the values needed.
 
         :param papi_reply: API reply from last executed PAPI command(s).
         :param stdout: stdout from last executed PAPI command(s).
         :param stderr: stderr from last executed PAPI command(s).
         :param ret_code: ret_code from last executed PAPI command(s).
+        :param requests: List of used PAPI requests. It is used while verifying
+            replies. If None, expected replies must be provided for verify_reply
+            and verify_replies methods.
         :type papi_reply: list
         :type stdout: str
         :type stderr: str
         :type ret_code: int
+        :type requests: list
         """
 
-        # API reply from last executed PAPI command(s)
+        # API reply from last executed PAPI command(s).
         self.reply = papi_reply
 
-        # stdout from last executed PAPI command(s)
+        # stdout from last executed PAPI command(s).
         self.stdout = stdout
 
         # stderr from last executed PAPI command(s).
         self.stderr = stderr
 
-        # return code from last executed PAPI command(s)
+        # return code from last executed PAPI command(s).
         self.ret_code = ret_code
 
+        # List of used PAPI requests.
+        self.requests = requests
+
+        # List of expected PAPI replies. It is used while verifying replies.
+        if self.requests:
+            self.expected_replies = \
+                ["{rqst}_reply".format(rqst=rqst) for rqst in self.requests]
+
     def __str__(self):
-        """Return string with human readable description of the group.
+        """Return string with human readable description of the PapiResponse.
 
         :returns: Readable description.
         :rtype: str
         """
-        return ("papi_reply={papi_reply} "
-                "stdout={stdout} "
-                "stderr={stderr} "
-                "ret_code={ret_code}".
+        return ("papi_reply={papi_reply},"
+                "stdout={stdout},"
+                "stderr={stderr},"
+                "ret_code={ret_code},"
+                "requests={requests}".
                 format(papi_reply=self.reply,
                        stdout=self.stdout,
                        stderr=self.stderr,
-                       ret_code=self.ret_code))
+                       ret_code=self.ret_code,
+                       requests=self.requests))
 
     def __repr__(self):
         """Return string executable as Python constructor call.
@@ -76,14 +101,104 @@ class PapiResponse(object):
         :returns: Executable constructor call.
         :rtype: str
         """
-        return ("PapiResponse(papi_reply={papi_reply} "
-                "stdout={stdout} "
-                "stderr={stderr} "
-                "ret_code={ret_code})".
-                format(papi_reply=self.reply,
-                       stdout=self.stdout,
-                       stderr=self.stderr,
-                       ret_code=self.ret_code))
+        return "PapiResponse({str})".format(str=str(self))
+
+    def verify_reply(self, cmd_reply=None, idx=0,
+                     err_msg="Failed to verify PAPI reply."):
+        """Verify and return data from the PAPI response.
+
+        Note: Use only with a simple request / reply command. In this case the
+        PAPI reply includes 'retval' which is checked in this method.
+
+        Use if PAPI response includes only one command reply.
+
+        Use it this way (preferred):
+
+        with PapiExecutor(node) as papi_exec:
+            data = papi_exec.add('show_version').execute_should_pass().\
+                verify_reply()
+
+        or if you must provide the expected reply (not recommended):
+
+        with PapiExecutor(node) as papi_exec:
+            data = papi_exec.add('show_version').execute_should_pass().\
+                verify_reply('show_version_reply')
+
+        :param cmd_reply: PAPI reply. If None, list of 'requests' should have
+            been provided to the __init__ method as pre-generated list of
+            replies is used in this method in this case.
+            The .execute* methods are providing the requests automatically.
+        :param idx: Index to PapiResponse.reply list.
+        :param err_msg: The message used if the verification fails.
+        :type cmd_reply: str
+        :type idx: int
+        :type err_msg: str or None
+        :returns: Verified data from PAPI response.
+        :rtype: dict
+        :raises AssertionError: If the PAPI return value is not 0, so the reply
+            is not valid.
+        :raises KeyError, IndexError: If the reply does not have expected
+            structure.
+        """
+        cmd_rpl = self.expected_replies[idx] if cmd_reply is None else cmd_reply
+
+        data = self.reply[idx]['api_reply'][cmd_rpl]
+        if data['retval'] != 0:
+            raise AssertionError("{msg}\nidx={idx}, cmd_reply={reply}".
+                                 format(msg=err_msg, idx=idx, reply=cmd_rpl))
+
+        return data
+
+    def verify_replies(self, cmd_replies=None,
+                       err_msg="Failed to verify PAPI reply."):
+        """Verify and return data from the PAPI response.
+
+        Note: Use only with request / reply commands. In this case each
+        PAPI reply includes 'retval' which is checked.
+
+        Use if PAPI response includes more than one command reply.
+
+        Use it this way:
+
+        with PapiExecutor(node) as papi_exec:
+            papi_exec.add(cmd1, **args1).add(cmd2, **args2).add(cmd2, **args3).\
+                execute_should_pass(err_msg).verify_replies()
+
+        or if you need the data from the PAPI response:
+
+        with PapiExecutor(node) as papi_exec:
+            data = papi_exec.add(cmd1, **args1).add(cmd2, **args2).\
+                add(cmd2, **args3).execute_should_pass(err_msg).verify_replies()
+
+        or if you must provide the list of expected replies (not recommended):
+
+        with PapiExecutor(node) as papi_exec:
+            data = papi_exec.add(cmd1, **args1).add(cmd2, **args2).\
+                add(cmd2, **args3).execute_should_pass(err_msg).\
+                verify_replies(cmd_replies=cmd_replies)
+
+        :param cmd_replies: List of PAPI command replies. If None, list of
+            'requests' should have been provided to the __init__ method as
+            pre-generated list of replies is used in this method in this case.
+            The .execute* methods are providing the requests automatically.
+        :param err_msg: The message used if the verification fails.
+        :type cmd_replies: list of str or None
+        :type err_msg: str
+        :returns: List of verified data from PAPI response.
+        :rtype list
+        :raises AssertionError: If the PAPI response does not include at least
+            one of specified command replies.
+        """
+        data = list()
+
+        cmd_rpls = self.expected_replies if cmd_replies is None else cmd_replies
+
+        if len(self.reply) != len(cmd_rpls):
+            raise AssertionError(err_msg)
+        for idx, cmd_reply in enumerate(cmd_rpls):
+            data.append(self.verify_reply(cmd_reply, idx, err_msg))
+
+        return data
 
 
 class PapiExecutor(object):
@@ -108,9 +223,6 @@ class PapiExecutor(object):
         # The list of PAPI commands to be executed on the node.
         self._api_command_list = list()
 
-        # The response on the PAPI commands.
-        self.response = PapiResponse()
-
         self._ssh = SSH()
 
     def __enter__(self):
@@ -197,7 +309,8 @@ class PapiExecutor(object):
         return PapiResponse(papi_reply=papi_reply,
                             stdout=stdout,
                             stderr=stderr,
-                            ret_code=ret_code)
+                            ret_code=ret_code,
+                            requests=[rqst["api_name"] for rqst in local_list])
 
     def execute_should_pass(self, err_msg="Failed to execute PAPI command.",
                             process_reply=True, ignore_errors=False,
@@ -205,9 +318,6 @@ class PapiExecutor(object):
         """Execute the PAPI commands and check the return code.
         Raise exception if the PAPI command(s) failed.
 
-        Note: There are two exceptions raised to distinguish two situations. If
-        not needed, re-implement using only RuntimeError.
-
         :param err_msg: The message used if the PAPI command(s) execution fails.
         :param process_reply: Indicate whether or not to process PAPI reply.
         :param ignore_errors: If true, the errors in the reply are ignored.
@@ -219,8 +329,7 @@ class PapiExecutor(object):
         :returns: Papi response including: papi reply, stdout, stderr and
             return code.
         :rtype: PapiResponse
-        :raises RuntimeError: If no PAPI command(s) executed.
-        :raises AssertionError: If PAPI command(s) execution passed.
+        :raises AssertionError: If PAPI command(s) execution failed.
         """
 
         response = self.execute(process_reply=process_reply,
@@ -240,9 +349,6 @@ class PapiExecutor(object):
 
         It does not return anything as we expect it fails.
 
-        Note: There are two exceptions raised to distinguish two situations. If
-        not needed, re-implement using only RuntimeError.
-
         :param err_msg: The message used if the PAPI command(s) execution fails.
         :param process_reply: Indicate whether or not to process PAPI reply.
         :param ignore_errors: If true, the errors in the reply are ignored.
@@ -251,7 +357,6 @@ class PapiExecutor(object):
         :type process_reply: bool
         :type ignore_errors: bool
         :type timeout: int
-        :raises RuntimeError: If no PAPI command(s) executed.
         :raises AssertionError: If PAPI command(s) execution passed.
         """