api: provide api definition over api

This patch allows a client to bootstrap itself by downloading the
JSON API definitions over the API itself.

This patch enables it for Python (probably need a dynamic language).
Call VPPApiClient with the new bootstrapapi=True parameter.

Example (Python):

from vpp_papi import VPPApiClient
vpp = VPPApiClient(bootstrapapi=True)
rv = vpp.connect("foobar")
assert rv == 0
print(f'SHOW VERSION: {vpp.api.show_version()}')
vpp.disconnect()

Type: feature
Change-Id: Id903fdccc82b2e22aa1994331d2c150253f2ccae
Signed-off-by: Ole Troan <otroan@employees.org>

3 files changed, 876 insertions, 21 deletions

diff --git a/src/vpp-api/python/setup.py b/src/vpp-api/python/setup.py
index 18637ba21b7..784013fc606 100644
--- a/src/vpp-api/python/setup.py
+++ b/src/vpp-api/python/setup.py
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import sys
 
 try:
     from setuptools import setup, find_packages
@@ -22,7 +21,7 @@ requirements = []
 
 setup(
     name="vpp_papi",
-    version="2.0.0",
+    version="2.1.0",
     description="VPP Python binding",
     author="Ole Troan",
     author_email="ot@cisco.com",
@@ -31,6 +30,7 @@ setup(
     test_suite="vpp_papi.tests",
     install_requires=requirements,
     packages=find_packages(),
+    package_data={"vpp_papi": ["data/*.json"]},
     long_description="""VPP Python language binding.""",
     zip_safe=True,
 )
diff --git a/src/vpp-api/python/vpp_papi/data/memclnt.api.json b/src/vpp-api/python/vpp_papi/data/memclnt.api.json
new file mode 100644
index 00000000000..1734cf12ab0
--- /dev/null
+++ b/src/vpp-api/python/vpp_papi/data/memclnt.api.json
@@ -0,0 +1,809 @@
+{
+    "types": [
+        [
+            "module_version",
+            [
+                "u32",
+                "major"
+            ],
+            [
+                "u32",
+                "minor"
+            ],
+            [
+                "u32",
+                "patch"
+            ],
+            [
+                "string",
+                "name",
+                64
+            ]
+        ],
+        [
+            "message_table_entry",
+            [
+                "u16",
+                "index"
+            ],
+            [
+                "string",
+                "name",
+                64
+            ]
+        ]
+    ],
+    "messages": [
+        [
+            "memclnt_create",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "ctx_quota"
+            ],
+            [
+                "u64",
+                "input_queue"
+            ],
+            [
+                "string",
+                "name",
+                64
+            ],
+            [
+                "u32",
+                "api_versions",
+                8
+            ],
+            {
+                "crc": "0x9c5e1c2f",
+                "options": {
+                    "deprecated": null
+                },
+                "comment": "/*\n * Create a client registration\n */"
+            }
+        ],
+        [
+            "memclnt_create_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "response"
+            ],
+            [
+                "u64",
+                "handle"
+            ],
+            [
+                "u32",
+                "index"
+            ],
+            [
+                "u64",
+                "message_table"
+            ],
+            {
+                "crc": "0x42ec4560",
+                "options": {
+                    "deprecated": null
+                }
+            }
+        ],
+        [
+            "memclnt_delete",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "index"
+            ],
+            [
+                "u64",
+                "handle"
+            ],
+            [
+                "bool",
+                "do_cleanup"
+            ],
+            {
+                "crc": "0x7e1c04e3",
+                "options": {},
+                "comment": "/*\n * Delete a client registration\n */"
+            }
+        ],
+        [
+            "memclnt_delete_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "i32",
+                "response"
+            ],
+            [
+                "u64",
+                "handle"
+            ],
+            {
+                "crc": "0x3d3b6312",
+                "options": {}
+            }
+        ],
+        [
+            "rx_thread_exit",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u8",
+                "dummy"
+            ],
+            {
+                "crc": "0xc3a3a452",
+                "options": {},
+                "comment": "/*\n * Client RX thread exit\n */"
+            }
+        ],
+        [
+            "memclnt_rx_thread_suspend",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u8",
+                "dummy"
+            ],
+            {
+                "crc": "0xc3a3a452",
+                "options": {},
+                "comment": "/*\n * Client RX thread suspend\n */"
+            }
+        ],
+        [
+            "memclnt_read_timeout",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u8",
+                "dummy"
+            ],
+            {
+                "crc": "0xc3a3a452",
+                "options": {},
+                "comment": "/*\n * Client read timeout\n */"
+            }
+        ],
+        [
+            "rpc_call",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "u64",
+                "function"
+            ],
+            [
+                "u8",
+                "multicast"
+            ],
+            [
+                "u8",
+                "need_barrier_sync"
+            ],
+            [
+                "u8",
+                "send_reply"
+            ],
+            [
+                "u32",
+                "data_len"
+            ],
+            [
+                "u8",
+                "data",
+                0,
+                "data_len"
+            ],
+            {
+                "crc": "0x7e8a2c95",
+                "options": {},
+                "comment": "/*\n * RPC\n */"
+            }
+        ],
+        [
+            "rpc_call_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "retval"
+            ],
+            {
+                "crc": "0xe8d4e804",
+                "options": {}
+            }
+        ],
+        [
+            "get_first_msg_id",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "string",
+                "name",
+                64
+            ],
+            {
+                "crc": "0xebf79a66",
+                "options": {},
+                "comment": "/*\n * Lookup message-ID base by name\n */"
+            }
+        ],
+        [
+            "get_first_msg_id_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "retval"
+            ],
+            [
+                "u16",
+                "first_msg_id"
+            ],
+            {
+                "crc": "0x7d337472",
+                "options": {}
+            }
+        ],
+        [
+            "api_versions",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            {
+                "crc": "0x51077d14",
+                "options": {},
+                "comment": "/*\n * Get API version table (includes built-in and plugins)\n */"
+            }
+        ],
+        [
+            "api_versions_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "retval"
+            ],
+            [
+                "u32",
+                "count"
+            ],
+            [
+                "vl_api_module_version_t",
+                "api_versions",
+                0,
+                "count"
+            ],
+            {
+                "crc": "0x5f0d99d6",
+                "options": {}
+            }
+        ],
+        [
+            "trace_plugin_msg_ids",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "string",
+                "plugin_name",
+                128
+            ],
+            [
+                "u16",
+                "first_msg_id"
+            ],
+            [
+                "u16",
+                "last_msg_id"
+            ],
+            {
+                "crc": "0xf476d3ce",
+                "options": {},
+                "comment": "/*\n * Trace the plugin message-id allocator\n * so we stand a chance of dealing with different sets of plugins\n * at api trace replay time\n */"
+            }
+        ],
+        [
+            "sockclnt_create",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "string",
+                "name",
+                64
+            ],
+            {
+                "crc": "0x455fb9c4",
+                "options": {},
+                "comment": "/*\n * Create a socket client registration.\n */"
+            }
+        ],
+        [
+            "sockclnt_create_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "response"
+            ],
+            [
+                "u32",
+                "index"
+            ],
+            [
+                "u16",
+                "count"
+            ],
+            [
+                "vl_api_message_table_entry_t",
+                "message_table",
+                0,
+                "count"
+            ],
+            {
+                "crc": "0x35166268",
+                "options": {}
+            }
+        ],
+        [
+            "sockclnt_delete",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "u32",
+                "index"
+            ],
+            {
+                "crc": "0x8ac76db6",
+                "options": {},
+                "comment": "/*\n * Delete a client registration\n */"
+            }
+        ],
+        [
+            "sockclnt_delete_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "response"
+            ],
+            {
+                "crc": "0x8f38b1ee",
+                "options": {}
+            }
+        ],
+        [
+            "sock_init_shm",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "u32",
+                "requested_size"
+            ],
+            [
+                "u8",
+                "nitems"
+            ],
+            [
+                "u64",
+                "configs",
+                0,
+                "nitems"
+            ],
+            {
+                "crc": "0x51646d92",
+                "options": {},
+                "comment": "/*\n * Initialize shm api over socket api\n */"
+            }
+        ],
+        [
+            "sock_init_shm_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "retval"
+            ],
+            {
+                "crc": "0xe8d4e804",
+                "options": {}
+            }
+        ],
+        [
+            "memclnt_keepalive",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            {
+                "crc": "0x51077d14",
+                "options": {},
+                "comment": "/*\n * Memory client ping / response\n * Only sent on inactive connections\n */"
+            }
+        ],
+        [
+            "memclnt_keepalive_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "retval"
+            ],
+            {
+                "crc": "0xe8d4e804",
+                "options": {}
+            }
+        ],
+        [
+            "control_ping",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            {
+                "crc": "0x51077d14",
+                "options": {},
+                "comment": "/** \\brief Control ping from client to api server request\n    @param client_index - opaque cookie to identify the sender\n    @param context - sender context, to match reply w/ request\n*/"
+            }
+        ],
+        [
+            "control_ping_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "retval"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "vpe_pid"
+            ],
+            {
+                "crc": "0xf6b0b8ca",
+                "options": {},
+                "comment": "/** \\brief Control ping from the client to the server response\n    @param client_index - opaque cookie to identify the sender\n    @param context - sender context, to match reply w/ request\n    @param retval - return code for the request\n    @param vpe_pid - the pid of the vpe, returned by the server\n*/"
+            }
+        ],
+        [
+            "memclnt_create_v2",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "ctx_quota"
+            ],
+            [
+                "u64",
+                "input_queue"
+            ],
+            [
+                "string",
+                "name",
+                64
+            ],
+            [
+                "u32",
+                "api_versions",
+                8
+            ],
+            [
+                "bool",
+                "keepalive",
+                {
+                    "default": "true"
+                }
+            ],
+            {
+                "crc": "0xc4bd4882",
+                "options": {}
+            }
+        ],
+        [
+            "memclnt_create_v2_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "response"
+            ],
+            [
+                "u64",
+                "handle"
+            ],
+            [
+                "u32",
+                "index"
+            ],
+            [
+                "u64",
+                "message_table"
+            ],
+            {
+                "crc": "0x42ec4560",
+                "options": {}
+            }
+        ],
+        [
+            "get_api_json",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "client_index"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            {
+                "crc": "0x51077d14",
+                "options": {}
+            }
+        ],
+        [
+            "get_api_json_reply",
+            [
+                "u16",
+                "_vl_msg_id"
+            ],
+            [
+                "u32",
+                "context"
+            ],
+            [
+                "i32",
+                "retval"
+            ],
+            [
+                "string",
+                "json",
+                0
+            ],
+            {
+                "crc": "0xea715b59",
+                "options": {}
+            }
+        ]
+    ],
+    "unions": [],
+    "enums": [],
+    "enumflags": [],
+    "services": {
+        "memclnt_rx_thread_suspend": {
+            "reply": "null"
+        },
+        "memclnt_read_timeout": {
+            "reply": "null"
+        },
+        "rx_thread_exit": {
+            "reply": "null"
+        },
+        "trace_plugin_msg_ids": {
+            "reply": "null"
+        },
+        "memclnt_create": {
+            "reply": "memclnt_create_reply"
+        },
+        "memclnt_delete": {
+            "reply": "memclnt_delete_reply"
+        },
+        "rpc_call": {
+            "reply": "rpc_call_reply"
+        },
+        "get_first_msg_id": {
+            "reply": "get_first_msg_id_reply"
+        },
+        "api_versions": {
+            "reply": "api_versions_reply"
+        },
+        "sockclnt_create": {
+            "reply": "sockclnt_create_reply"
+        },
+        "sockclnt_delete": {
+            "reply": "sockclnt_delete_reply"
+        },
+        "sock_init_shm": {
+            "reply": "sock_init_shm_reply"
+        },
+        "memclnt_keepalive": {
+            "reply": "memclnt_keepalive_reply"
+        },
+        "control_ping": {
+            "reply": "control_ping_reply"
+        },
+        "memclnt_create_v2": {
+            "reply": "memclnt_create_v2_reply"
+        },
+        "get_api_json": {
+            "reply": "get_api_json_reply"
+        }
+    },
+    "options": {
+        "version": "2.1.0"
+    },
+    "aliases": {},
+    "vl_api_version": "0xb197c551",
+    "imports": [],
+    "counters": [],
+    "paths": []
+}
diff --git a/src/vpp-api/python/vpp_papi/vpp_papi.py b/src/vpp-api/python/vpp_papi/vpp_papi.py
index 5a9e0a70c64..30c00cd8dd3 100644
--- a/src/vpp-api/python/vpp_papi/vpp_papi.py
+++ b/src/vpp-api/python/vpp_papi/vpp_papi.py
@@ -18,7 +18,6 @@ from __future__ import print_function
 from __future__ import absolute_import
 import ctypes
 import ipaddress
-import sys
 import multiprocessing as mp
 import os
 import queue
@@ -30,6 +29,7 @@ import fnmatch
 import weakref
 import atexit
 import time
+import pkg_resources
 from .vpp_format import verify_enum_hint
 from .vpp_serializer import VPPType, VPPEnumType, VPPEnumFlagType, VPPUnionType
 from .vpp_serializer import VPPMessage, vpp_get_type, VPPTypeAlias
@@ -281,15 +281,28 @@ class VPPApiJSONFiles:
 
     @classmethod
     def process_json_file(self, apidef_file):
-        return self._process_json(apidef_file.read())
+        api = json.load(apidef_file)
+        return self._process_json(api)
 
     @classmethod
     def process_json_str(self, json_str):
-        return self._process_json(json_str)
+        api = json.loads(json_str)
+        return self._process_json(api)
+
+    @classmethod
+    def process_json_array_str(self, json_str):
+        services = {}
+        messages = {}
+
+        apis = json.loads(json_str)
+        for a in apis:
+            m, s = self._process_json(a)
+            messages.update(m)
+            services.update(s)
+        return messages, services
 
     @staticmethod
-    def _process_json(json_str):  # -> Tuple[Dict, Dict]
-        api = json.loads(json_str)
+    def _process_json(api):  # -> Tuple[Dict, Dict]
         types = {}
         services = {}
         messages = {}
@@ -373,7 +386,6 @@ class VPPApiJSONFiles:
                 try:
                     messages[m[0]] = VPPMessage(m[0], m[1:])
                 except VPPNotImplementedError:
-                    ### OLE FIXME
                     logger.error("Not implemented error for {}".format(m[0]))
         except KeyError:
             pass
@@ -435,6 +447,7 @@ class VPPApiClient:
         read_timeout=5,
         use_socket=True,
         server_address="/run/vpp/api.sock",
+        bootstrapapi=False,
     ):
         """Create a VPP API object.
 
@@ -472,25 +485,37 @@ class VPPApiClient:
         self.server_address = server_address
         self._apifiles = apifiles
         self.stats = {}
+        self.bootstrapapi = bootstrapapi
 
-        if self.apidir is None and hasattr(self.__class__, "apidir"):
-            # Keep supporting the old style of providing apidir.
-            self.apidir = self.__class__.apidir
-        try:
-            self.apifiles, self.messages, self.services = VPPApiJSONFiles.load_api(
-                apifiles, self.apidir
+        if not bootstrapapi:
+            if self.apidir is None and hasattr(self.__class__, "apidir"):
+                # Keep supporting the old style of providing apidir.
+                self.apidir = self.__class__.apidir
+            try:
+                self.apifiles, self.messages, self.services = VPPApiJSONFiles.load_api(
+                    apifiles, self.apidir
+                )
+            except VPPRuntimeError as e:
+                if testmode:
+                    self.apifiles = []
+                else:
+                    raise e
+        else:
+            # Bootstrap the API (memclnt.api bundled with VPP PAPI)
+            resource_path = "/".join(("data", "memclnt.api.json"))
+            file_content = pkg_resources.resource_string(__name__, resource_path)
+            self.messages, self.services = VPPApiJSONFiles.process_json_str(
+                file_content
             )
-        except VPPRuntimeError as e:
-            if testmode:
-                self.apifiles = []
-            else:
-                raise e
 
         # Basic sanity check
         if len(self.messages) == 0 and not testmode:
             raise VPPValueError(1, "Missing JSON message definitions")
-        if not (verify_enum_hint(VppEnum.vl_api_address_family_t)):
-            raise VPPRuntimeError("Invalid address family hints. " "Cannot continue.")
+        if not bootstrapapi:
+            if not (verify_enum_hint(VppEnum.vl_api_address_family_t)):
+                raise VPPRuntimeError(
+                    "Invalid address family hints. " "Cannot continue."
+                )
 
         self.transport = VppTransport(
             self, read_timeout=read_timeout, server_address=server_address
@@ -573,6 +598,21 @@ class VPPApiClient:
             else:
                 self.logger.debug("No such message type or failed CRC checksum: %s", n)
 
+    def get_api_definitions(self):
+        """get_api_definition. Bootstrap from the embedded memclnt.api.json file."""
+
+        # Bootstrap so we can call the get_api_json function
+        self._register_functions(do_async=False)
+
+        r = self.api.get_api_json()
+        if r.retval != 0:
+            raise VPPApiError("Failed to load API definitions from VPP")
+
+        # Process JSON
+        m, s = VPPApiJSONFiles.process_json_array_str(r.json)
+        self.messages.update(m)
+        self.services.update(s)
+
     def connect_internal(self, name, msg_handler, chroot_prefix, rx_qlen, do_async):
         pfx = chroot_prefix.encode("utf-8") if chroot_prefix else None
 
@@ -580,6 +620,10 @@ class VPPApiClient:
         if rv != 0:
             raise VPPIOError(2, "Connect failed")
         self.vpp_dictionary_maxid = self.transport.msg_table_max_index()
+
+        # Register functions
+        if self.bootstrapapi:
+            self.get_api_definitions()
         self._register_functions(do_async=do_async)
 
         # Initialise control ping
@@ -588,6 +632,7 @@ class VPPApiClient:
             ("control_ping" + "_" + crc[2:])
         )
         self.control_ping_msgdef = self.messages["control_ping"]
+
         if self.async_thread:
             self.event_thread = threading.Thread(target=self.thread_msg_handler)
             self.event_thread.daemon = True
@@ -659,6 +704,7 @@ class VPPApiClient:
         )
 
         (i, ci, context), size = header.unpack(msg, 0)
+
         if self.id_names[i] == "rx_thread_exit":
             return