api: vppapitrace JSON/API trace converter

usage: vppapitrace.py [-h] [--debug] [--apidir APIDIR] {convert,replay} ...

optional arguments:
  -h, --help        show this help message and exit
  --debug           enable debug mode
  --apidir APIDIR   Location of JSON API definitions

subcommands:
  valid subcommands

  {convert,replay}  additional help
    convert         Convert API trace to JSON or Python and back
    replay          Replay messages to running VPP instance

To convert an API trace file to JSON:
vppapitrace convert /tmp/api.trace trace.json

To convert an (edited) JSON file back to API trace for replay:
vppapitrace convert trace.json api-edited.trace

To generate a Python file that can be replayed:
vppapitrace convert /tmp/api.trace trace.py
vppapitrace convert trace.json trace.py

Replay it to a running VPP instance:
vppapitrace replay --socket /tmp/api.trace

In VPP that file can be replayed with:
vpp# api trace replay api-edited.trace

This patch also modifies the API binary trace format, to include the
message id to message name table.

Change-Id: Ie6441efb53c1c93c9f778f6ae9c1758bccc8dd87
Type: refactor
Signed-off-by: Ole Troan <ot@cisco.com>

2 files changed, 153 insertions, 141 deletions

diff --git a/src/vpp-api/python/vpp_papi/__init__.py b/src/vpp-api/python/vpp_papi/__init__.py
index 957468a5baf..e1b77811aef 100644
--- a/src/vpp-api/python/vpp_papi/__init__.py
+++ b/src/vpp-api/python/vpp_papi/__init__.py
@@ -2,6 +2,7 @@ from .vpp_papi import FuncWrapper, VPP, VppApiDynamicMethodHolder  # noqa: F401
 from .vpp_papi import VppEnum, VppEnumType  # noqa: F401
 from .vpp_papi import VPPIOError, VPPRuntimeError, VPPValueError  # noqa: F401
 from .vpp_papi import VPPApiClient  # noqa: F401
+from .vpp_papi import VPPApiJSONFiles # noqa: F401
 from . macaddress import MACAddress, mac_pton, mac_ntop  # noqa: F401
 
 # sorted lexicographically
diff --git a/src/vpp-api/python/vpp_papi/vpp_papi.py b/src/vpp-api/python/vpp_papi/vpp_papi.py
index 818a55f52f3..b3f2a156939 100644
--- a/src/vpp-api/python/vpp_papi/vpp_papi.py
+++ b/src/vpp-api/python/vpp_papi/vpp_papi.py
@@ -112,29 +112,133 @@ class VPPRuntimeError(RuntimeError):
 class VPPValueError(ValueError):
     pass
 
+class VPPApiJSONFiles(object):
+    @classmethod
+    def find_api_dir(cls, dirs):
+        """Attempt to find the best directory in which API definition
+        files may reside. If the value VPP_API_DIR exists in the environment
+        then it is first on the search list. If we're inside a recognized
+        location in a VPP source tree (src/scripts and src/vpp-api/python)
+        then entries from there to the likely locations in build-root are
+        added. Finally the location used by system packages is added.
 
-class VPPApiClient(object):
-    """VPP interface.
+        :returns: A single directory name, or None if no such directory
+            could be found.
+        """
 
-    This class provides the APIs to VPP.  The APIs are loaded
-    from provided .api.json files and makes functions accordingly.
-    These functions are documented in the VPP .api files, as they
-    are dynamically created.
+        # perhaps we're in the 'src/scripts' or 'src/vpp-api/python' dir;
+        # in which case, plot a course to likely places in the src tree
+        import __main__ as main
+        if hasattr(main, '__file__'):
+            # get the path of the calling script
+            localdir = os.path.dirname(os.path.realpath(main.__file__))
+        else:
+            # use cwd if there is no calling script
+            localdir = os.getcwd()
+        localdir_s = localdir.split(os.path.sep)
 
-    Additionally, VPP can send callback messages; this class
-    provides a means to register a callback function to receive
-    these messages in a background thread.
-    """
-    apidir = None
-    VPPApiError = VPPApiError
-    VPPRuntimeError = VPPRuntimeError
-    VPPValueError = VPPValueError
-    VPPNotImplementedError = VPPNotImplementedError
-    VPPIOError = VPPIOError
+        def dmatch(dir):
+            """Match dir against right-hand components of the script dir"""
+            d = dir.split('/')  # param 'dir' assumes a / separator
+            length = len(d)
+            return len(localdir_s) > length and localdir_s[-length:] == d
+
+        def sdir(srcdir, variant):
+            """Build a path from srcdir to the staged API files of
+            'variant'  (typically '' or '_debug')"""
+            # Since 'core' and 'plugin' files are staged
+            # in separate directories, we target the parent dir.
+            return os.path.sep.join((
+                srcdir,
+                'build-root',
+                'install-vpp%s-native' % variant,
+                'vpp',
+                'share',
+                'vpp',
+                'api',
+            ))
+
+        srcdir = None
+        if dmatch('src/scripts'):
+            srcdir = os.path.sep.join(localdir_s[:-2])
+        elif dmatch('src/vpp-api/python'):
+            srcdir = os.path.sep.join(localdir_s[:-3])
+        elif dmatch('test'):
+            # we're apparently running tests
+            srcdir = os.path.sep.join(localdir_s[:-1])
+
+        if srcdir:
+            # we're in the source tree, try both the debug and release
+            # variants.
+            dirs.append(sdir(srcdir, '_debug'))
+            dirs.append(sdir(srcdir, ''))
+
+        # Test for staged copies of the scripts
+        # For these, since we explicitly know if we're running a debug versus
+        # release variant, target only the relevant directory
+        if dmatch('build-root/install-vpp_debug-native/vpp/bin'):
+            srcdir = os.path.sep.join(localdir_s[:-4])
+            dirs.append(sdir(srcdir, '_debug'))
+        if dmatch('build-root/install-vpp-native/vpp/bin'):
+            srcdir = os.path.sep.join(localdir_s[:-4])
+            dirs.append(sdir(srcdir, ''))
+
+        # finally, try the location system packages typically install into
+        dirs.append(os.path.sep.join(('', 'usr', 'share', 'vpp', 'api')))
+
+        # check the directories for existence; first one wins
+        for dir in dirs:
+            if os.path.isdir(dir):
+                return dir
+
+        return None
+
+    @classmethod
+    def find_api_files(cls, api_dir=None, patterns='*'):
+        """Find API definition files from the given directory tree with the
+        given pattern. If no directory is given then find_api_dir() is used
+        to locate one. If no pattern is given then all definition files found
+        in the directory tree are used.
+
+        :param api_dir: A directory tree in which to locate API definition
+            files; subdirectories are descended into.
+            If this is None then find_api_dir() is called to discover it.
+        :param patterns: A list of patterns to use in each visited directory
+            when looking for files.
+            This can be a list/tuple object or a comma-separated string of
+            patterns. Each value in the list will have leading/trialing
+            whitespace stripped.
+            The pattern specifies the first part of the filename, '.api.json'
+            is appended.
+            The results are de-duplicated, thus overlapping patterns are fine.
+            If this is None it defaults to '*' meaning "all API files".
+        :returns: A list of file paths for the API files found.
+        """
+        if api_dir is None:
+            api_dir = cls.find_api_dir([])
+            if api_dir is None:
+                raise VPPApiError("api_dir cannot be located")
+
+        if isinstance(patterns, list) or isinstance(patterns, tuple):
+            patterns = [p.strip() + '.api.json' for p in patterns]
+        else:
+            patterns = [p.strip() + '.api.json' for p in patterns.split(",")]
+
+        api_files = []
+        for root, dirnames, files in os.walk(api_dir):
+            # iterate all given patterns and de-dup the result
+            files = set(sum([fnmatch.filter(files, p) for p in patterns], []))
+            for filename in files:
+                api_files.append(os.path.join(root, filename))
 
+        return api_files
+
+    @classmethod
     def process_json_file(self, apidef_file):
         api = json.load(apidef_file)
         types = {}
+        services = {}
+        messages = {}
         for t in api['enums']:
             t[0] = 'vl_api_' + t[0] + '_t'
             types[t[0]] = {'type': 'enum', 'data': t}
@@ -146,7 +250,7 @@ class VPPApiClient(object):
             types[t[0]] = {'type': 'type', 'data': t}
         for t, v in api['aliases'].items():
             types['vl_api_' + t + '_t'] = {'type': 'alias', 'data': v}
-        self.services.update(api['services'])
+        services.update(api['services'])
 
         i = 0
         while True:
@@ -184,9 +288,31 @@ class VPPApiClient(object):
 
         for m in api['messages']:
             try:
-                self.messages[m[0]] = VPPMessage(m[0], m[1:])
+                messages[m[0]] = VPPMessage(m[0], m[1:])
             except VPPNotImplementedError:
+                ### OLE FIXME
                 self.logger.error('Not implemented error for {}'.format(m[0]))
+        return messages, services
+
+class VPPApiClient(object):
+    """VPP interface.
+
+    This class provides the APIs to VPP.  The APIs are loaded
+    from provided .api.json files and makes functions accordingly.
+    These functions are documented in the VPP .api files, as they
+    are dynamically created.
+
+    Additionally, VPP can send callback messages; this class
+    provides a means to register a callback function to receive
+    these messages in a background thread.
+    """
+    apidir = None
+    VPPApiError = VPPApiError
+    VPPRuntimeError = VPPRuntimeError
+    VPPValueError = VPPValueError
+    VPPNotImplementedError = VPPNotImplementedError
+    VPPIOError = VPPIOError
+
 
     def __init__(self, apifiles=None, testmode=False, async_thread=True,
                  logger=None, loglevel=None,
@@ -236,7 +362,7 @@ class VPPApiClient(object):
         if not apifiles:
             # Pick up API definitions from default directory
             try:
-                apifiles = self.find_api_files()
+                apifiles = VPPApiJSONFiles.find_api_files(self.apidir)
             except RuntimeError:
                 # In test mode we don't care that we can't find the API files
                 if testmode:
@@ -246,7 +372,9 @@ class VPPApiClient(object):
 
         for file in apifiles:
             with open(file) as apidef_file:
-                self.process_json_file(apidef_file)
+                m, s = VPPApiJSONFiles.process_json_file(apidef_file)
+                self.messages.update(m)
+                self.services.update(s)
 
         self.apifiles = apifiles
 
@@ -259,6 +387,10 @@ class VPPApiClient(object):
         # Make sure we allow VPP to clean up the message rings.
         atexit.register(vpp_atexit, weakref.ref(self))
 
+    def get_function(self, name):
+        return getattr(self._api, name)
+
+
     class ContextId(object):
         """Multiprocessing-safe provider of unique context IDs."""
         def __init__(self):
@@ -275,127 +407,6 @@ class VPPApiClient(object):
     def get_type(self, name):
         return vpp_get_type(name)
 
-    @classmethod
-    def find_api_dir(cls):
-        """Attempt to find the best directory in which API definition
-        files may reside. If the value VPP_API_DIR exists in the environment
-        then it is first on the search list. If we're inside a recognized
-        location in a VPP source tree (src/scripts and src/vpp-api/python)
-        then entries from there to the likely locations in build-root are
-        added. Finally the location used by system packages is added.
-
-        :returns: A single directory name, or None if no such directory
-            could be found.
-        """
-        dirs = [cls.apidir] if cls.apidir else []
-
-        # perhaps we're in the 'src/scripts' or 'src/vpp-api/python' dir;
-        # in which case, plot a course to likely places in the src tree
-        import __main__ as main
-        if hasattr(main, '__file__'):
-            # get the path of the calling script
-            localdir = os.path.dirname(os.path.realpath(main.__file__))
-        else:
-            # use cwd if there is no calling script
-            localdir = os.getcwd()
-        localdir_s = localdir.split(os.path.sep)
-
-        def dmatch(dir):
-            """Match dir against right-hand components of the script dir"""
-            d = dir.split('/')  # param 'dir' assumes a / separator
-            length = len(d)
-            return len(localdir_s) > length and localdir_s[-length:] == d
-
-        def sdir(srcdir, variant):
-            """Build a path from srcdir to the staged API files of
-            'variant'  (typically '' or '_debug')"""
-            # Since 'core' and 'plugin' files are staged
-            # in separate directories, we target the parent dir.
-            return os.path.sep.join((
-                srcdir,
-                'build-root',
-                'install-vpp%s-native' % variant,
-                'vpp',
-                'share',
-                'vpp',
-                'api',
-            ))
-
-        srcdir = None
-        if dmatch('src/scripts'):
-            srcdir = os.path.sep.join(localdir_s[:-2])
-        elif dmatch('src/vpp-api/python'):
-            srcdir = os.path.sep.join(localdir_s[:-3])
-        elif dmatch('test'):
-            # we're apparently running tests
-            srcdir = os.path.sep.join(localdir_s[:-1])
-
-        if srcdir:
-            # we're in the source tree, try both the debug and release
-            # variants.
-            dirs.append(sdir(srcdir, '_debug'))
-            dirs.append(sdir(srcdir, ''))
-
-        # Test for staged copies of the scripts
-        # For these, since we explicitly know if we're running a debug versus
-        # release variant, target only the relevant directory
-        if dmatch('build-root/install-vpp_debug-native/vpp/bin'):
-            srcdir = os.path.sep.join(localdir_s[:-4])
-            dirs.append(sdir(srcdir, '_debug'))
-        if dmatch('build-root/install-vpp-native/vpp/bin'):
-            srcdir = os.path.sep.join(localdir_s[:-4])
-            dirs.append(sdir(srcdir, ''))
-
-        # finally, try the location system packages typically install into
-        dirs.append(os.path.sep.join(('', 'usr', 'share', 'vpp', 'api')))
-
-        # check the directories for existence; first one wins
-        for dir in dirs:
-            if os.path.isdir(dir):
-                return dir
-
-        return None
-
-    @classmethod
-    def find_api_files(cls, api_dir=None, patterns='*'):
-        """Find API definition files from the given directory tree with the
-        given pattern. If no directory is given then find_api_dir() is used
-        to locate one. If no pattern is given then all definition files found
-        in the directory tree are used.
-
-        :param api_dir: A directory tree in which to locate API definition
-            files; subdirectories are descended into.
-            If this is None then find_api_dir() is called to discover it.
-        :param patterns: A list of patterns to use in each visited directory
-            when looking for files.
-            This can be a list/tuple object or a comma-separated string of
-            patterns. Each value in the list will have leading/trialing
-            whitespace stripped.
-            The pattern specifies the first part of the filename, '.api.json'
-            is appended.
-            The results are de-duplicated, thus overlapping patterns are fine.
-            If this is None it defaults to '*' meaning "all API files".
-        :returns: A list of file paths for the API files found.
-        """
-        if api_dir is None:
-            api_dir = cls.find_api_dir()
-            if api_dir is None:
-                raise VPPApiError("api_dir cannot be located")
-
-        if isinstance(patterns, list) or isinstance(patterns, tuple):
-            patterns = [p.strip() + '.api.json' for p in patterns]
-        else:
-            patterns = [p.strip() + '.api.json' for p in patterns.split(",")]
-
-        api_files = []
-        for root, dirnames, files in os.walk(api_dir):
-            # iterate all given patterns and de-dup the result
-            files = set(sum([fnmatch.filter(files, p) for p in patterns], []))
-            for filename in files:
-                api_files.append(os.path.join(root, filename))
-
-        return api_files
-
     @property
     def api(self):
         if not hasattr(self, "_api"):