Add client-side msg_name_and_crc -> msg_index table

vppapigen now generates per-message crcs. Verified that whitespace
and real changes in message A don't change the crc for message B, etc.

Fixed the sample and flowperpkt plugins to participate. Others need
the same treatment. They don't build due to python/java language binding
build issues.

To use the scheme:

Client connects as usual.

Then call: u32 vl_api_get_msg_index(char * name_and_crc)
name_and_crc is a string like: "flowperpkt_tx_interface_add_del_753301f3",
aka the message name with _%08x <expected crc> appended.

Try these vpp-api-test commands to play with it:

vat# dump_msg_api_table
     <snip>
 [366]: punt_reply_cca27fbe
 [367]: ipsec_spd_dump_5e9ae88e
 [368]: ipsec_spd_details_6f7821b0
 [369]: sample_macswap_enable_disable_0f2813e2
 [370]: sample_macswap_enable_disable_reply_476738e5
 [371]: flowperpkt_tx_interface_add_del_753301f3
 [372]: flowperpkt_tx_interface_add_del_reply_d47e6e0b

vat# get_msg_id sample_macswap_enable_disable_reply_476738e5
 'sample_macswap_enable_disable_reply_476738e5' has message index 370

vat# get_msg_id sample_macswap_enable_disable_reply_476738e3
 'sample_macswap_enable_disable_reply_476738e3' not found

CRCs may vary, etc.

vppapigen is used to build a set of JSON representations
of each API file from vpp-api/Makefile.am and that is in
turn used by each language binding (Java, Python, Lua).

Change-Id: I3d64582e779dac5f20cddec79c562c288d8fd9c6
Signed-off-by: Dave Barach <dave@barachs.net>
Signed-off-by: Ole Troan <ot@cisco.com>

6 files changed, 98 insertions, 393 deletions

diff --git a/vpp-api/python/Makefile.am b/vpp-api/python/Makefile.am
index 59b1b9220bd..d2c3fb5dd4e 100644
--- a/vpp-api/python/Makefile.am
+++ b/vpp-api/python/Makefile.am
@@ -52,7 +52,8 @@ install-exec-local: $(srcdir)/vpp_papi/vpe.py $(srcdir)/vpp_papi/memclnt.py
 	cd $(srcdir);							\
 	mkdir -p $(prefix)/lib/python2.7/site-packages; 		\
 	PYTHONUSERBASE=$(prefix)					\
-	python setup.py build_ext -L $(prefix)/lib64 install --user
+	python setup.py build_ext -L $(prefix)/lib64 \
+	-I $(prefix)/../vppinfra/include/ install --user
 
 #
 # Test client
diff --git a/vpp-api/python/pneum/api-gen.py b/vpp-api/python/pneum/api-gen.py
deleted file mode 100755
index 445683523de..00000000000
--- a/vpp-api/python/pneum/api-gen.py
+++ /dev/null
@@ -1,351 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import argparse, sys, os, importlib, pprint
-
-parser = argparse.ArgumentParser(description='VPP Python API generator')
-parser.add_argument('-i', action="store", dest="inputfile")
-parser.add_argument('-c', '--cfile', action="store")
-args = parser.parse_args()
-
-sys.path.append(".")
-
-inputfile = args.inputfile.replace('.py', '')
-cfg = importlib.import_module(inputfile, package=None)
-
-# https://docs.python.org/3/library/struct.html
-format_struct = {'u8': 'B',
-                 'u16' : 'H',
-                 'u32' : 'I',
-                 'i32' : 'i',
-                 'u64' : 'Q',
-                 'f64' : 'd',
-                 'vl_api_fib_path_t' : 'IIBBBBBBBBBBBBBBBBBBBBB',
-                 'vl_api_ip4_fib_counter_t' : 'IBQQ',
-                 'vl_api_ip6_fib_counter_t' : 'QQBQQ',
-                 'vl_api_lisp_adjacency_t' : 'B' * 35,
-                 };
-#
-# NB: If new types are introduced in vpe.api, these must be updated.
-#
-type_size = {'u8':   1,
-             'u16' : 2,
-             'u32' : 4,
-             'i32' : 4,
-             'u64' : 8,
-             'f64' : 8,
-             'vl_api_fib_path_t' : 29,
-             'vl_api_ip4_fib_counter_t' : 21,
-             'vl_api_ip6_fib_counter_t' : 33,
-             'vl_api_lisp_adjacency_t' : 35,
-};
-
-def get_args(t):
-    argslist = []
-    for i in t:
-        if i[1][0] == '_':
-            argslist.append(i[1][1:])
-        else:
-            argslist.append(i[1])
-
-    return argslist
-
-def get_pack(t):
-    zeroarray = False
-    bytecount = 0
-    pack = '>'
-    tup = u''
-    j = -1
-    for i in t:
-        j += 1
-        if len(i) is 3 or len(i) is 4:  # TODO: add support for variable length arrays (VPP-162)
-            size = type_size[i[0]]
-            bytecount += size * int(i[2])
-            # Check if we have a zero length array
-            if i[2] == '0':
-                tup += 'msg[' + str(bytecount) + ':],'
-                zeroarray = True
-                continue
-            if size == 1:
-                n = i[2] * size
-                pack += str(n) + 's'
-                tup += 'tr[' + str(j) + '],'
-                continue
-            pack += format_struct[i[0]] * int(i[2])
-            tup += 'tr[' + str(j) + ':' + str(j + int(i[2])) + '],'
-            j += int(i[2]) - 1
-        else:
-            bytecount += type_size[i[0]]
-            pack += format_struct[i[0]]
-            tup += 'tr[' + str(j) + '],'
-    return pack, bytecount, tup, zeroarray
-
-def get_reply_func(f):
-    if f['name']+'_reply' in func_name:
-        return func_name[f['name']+'_reply']
-    if f['name'].find('_dump') > 0:
-        r = f['name'].replace('_dump','_details')
-        if r in func_name:
-            return func_name[r]
-    return None
-
-def get_enums():
-    # Read enums from stdin
-    enums_by_name = {}
-    enums_by_index = {}
-    i = 1
-    for l in sys.stdin:
-        l = l.replace(',\n','')
-        print l, "=", i
-
-        l = l.replace('VL_API_','').lower()
-        enums_by_name[l] = i
-        enums_by_index[i] = l
-
-        i += 1
-    return enums_by_name, enums_by_index
-
-def get_definitions():
-    # Pass 1
-    func_list = []
-    func_name = {}
-    i = 1
-    for a in cfg.messages:
-        pack, packlen, tup, zeroarray = get_pack(a[1:])
-        func_name[a[0]] = dict([('name', a[0]), ('pack', pack), ('packlen', packlen), ('tup', tup), ('args', get_args(a[1:])),
-                                ('zeroarray', zeroarray)])
-        func_list.append(func_name[a[0]])  # Indexed by name
-    return func_list, func_name
-
-def generate_c_macros(func_list, enums_by_name):
-    file = open(args.cfile, 'w+')
-    print >>file, "#define foreach_api_msg \\"
-    for f in func_list:
-        if not f['name'] in enums_by_name:
-            continue
-        print >>file, "_(" + f['name'].upper() + ", " + f['name'] + ") \\"
-    print >>file, '''
-void pneum_set_handlers(void) {
-#define _(N,n)							\\
-  api_func_table[VL_API_##N] = sizeof(vl_api_##n##_t);
-  foreach_api_msg;
-#undef _
-}
-    '''
-
-#
-# Print array with a hash of 'decode' and 'multipart'
-# Simplify to do only decode for now. And deduce multipart from _dump?
-#
-def decode_function_print(name, args, pack, packlen, tup):
-
-    print(u'def ' + name + u'_decode(msg):')
-    print(u"    n = namedtuple('" + name + "', '" + ', '.join(args) + "')" +
-    '''
-    if not n:
-        return None
-    ''')
-    print(u"    tr = unpack('" + pack + "', msg[:" + str(packlen) + "])")
-    print(u"    r = n._make((" + tup + "))" +
-    '''
-    if not r:
-        return None
-    return r
-    ''')
-
-def function_print(name, id, args, pack, multipart, zeroarray):
-    if len(args) < 4:
-        print "def", name + "(async = False):"
-    else:
-        print "def", name + "(" + ', '.join(args[3:]) + ", async = False):"
-    print "    global waiting_for_reply"
-    print "    context = get_context(" + id + ")"
-
-    print '''
-    results[context] = {}
-    results[context]['e'] = threading.Event()
-    results[context]['e'].clear()
-    results[context]['r'] = []
-    waiting_for_reply = True
-    '''
-    if multipart == True:
-        print "    results[context]['m'] = True"
-
-    if zeroarray == True:
-        print "    vpp_api.write(pack('" + pack + "', " + id + ", 0, context, " + ', '.join(args[3:-1]) + ") + " + args[-1] + ")"
-    else:
-        print "    vpp_api.write(pack('" + pack + "', " + id + ", 0, context, " + ', '.join(args[3:]) + "))"
-
-    if multipart == True:
-        print "    vpp_api.write(pack('>HII', VL_API_CONTROL_PING, 0, context))"
-
-    print '''
-    if not async:
-        results[context]['e'].wait(5)
-        return results[context]['r']
-    return context
-    '''
-
-#
-# Should dynamically create size
-#
-def api_table_print (name, msg_id):
-    f = name + '_decode'
-    print('api_func_table[' + msg_id + '] = ' + f)
-
-#
-# Generate the main Python file
-#
-
-print '''
-
-#
-# AUTO-GENERATED FILE. PLEASE DO NOT EDIT.
-#
-import sys, time, threading, signal, os, logging
-from struct import *
-from collections import namedtuple
-
-#
-# Import C API shared object
-#
-import vpp_api
-
-context = 0
-results = {}
-waiting_for_reply = False
-
-#
-# XXX: Make this return a unique number
-#
-def get_context(id):
-    global context
-    context += 1
-    return context
-
-def msg_handler(msg):
-    global result, context, event_callback, waiting_for_reply
-    if not msg:
-        logging.warning('vpp_api.read failed')
-        return
-
-    id = unpack('>H', msg[0:2])
-    logging.debug('Received message', id[0])
-    if id[0] == VL_API_RX_THREAD_EXIT:
-        logging.info("We got told to leave")
-        return;
-
-    #
-    # Decode message and returns a tuple.
-    #
-    logging.debug('api_func', api_func_table[id[0]])
-    r = api_func_table[id[0]](msg)
-    if not r:
-        logging.warning('Message decode failed', id[0])
-        return
-
-    if 'context' in r._asdict():
-        if r.context > 0:
-            context = r.context
-
-    #
-    # XXX: Call provided callback for event
-    # Are we guaranteed to not get an event during processing of other messages?
-    # How to differentiate what's a callback message and what not? Context = 0?
-    #
-    logging.debug('R:', context, r, waiting_for_reply)
-    if waiting_for_reply == False:
-        event_callback(r)
-        return
-
-    #
-    # Collect results until control ping
-    #
-    if id[0] == VL_API_CONTROL_PING_REPLY:
-        results[context]['e'].set()
-        waiting_for_reply = False
-        return
-    if not context in results:
-        logging.warning('Not expecting results for this context', context)
-        return
-    if 'm' in results[context]:
-        results[context]['r'].append(r)
-        return
-
-    results[context]['r'] = r
-    results[context]['e'].set()
-    waiting_for_reply = False
-
-def connect(name):
-    signal.alarm(3) # 3 second
-    rv = vpp_api.connect(name, msg_handler)
-    signal.alarm(0)
-    logging.info("Connect:", rv)
-    return rv
-
-def disconnect():
-    rv = vpp_api.disconnect()
-    logging.info("Disconnected")
-    return rv
-
-def register_event_callback(callback):
-    global event_callback
-    event_callback = callback
-'''
-
-enums_by_name, enums_by_index = get_enums()
-func_list, func_name = get_definitions()
-
-#
-# Not needed with the new msg_size field.
-# generate_c_macros(func_list, enums_by_name)
-#
-
-pp = pprint.PrettyPrinter(indent=4)
-#print 'enums_by_index =', pp.pprint(enums_by_index)
-#print 'func_name =', pp.pprint(func_name)
-
-# Pass 2
-
-#
-# 1) The VPE API lacks a clear definition of what messages are reply messages
-# 2) Length is missing, and has to be pre-known or in case of variable sized ones calculated per message type
-#
-for f in func_list:
-    #if f['name'].find('_reply') > 0 or f['name'].find('_details') > 0:
-    decode_function_print(f['name'], f['args'], f['pack'], f['packlen'], f['tup'])
-
-    #r = get_reply_func(f)
-    #if not r:
-    #    #
-    #    # XXX: Functions here are not taken care of. E.g. events
-    #    #
-    #    print('Missing function', f)
-    #    continue
-
-    if f['name'].find('_dump') > 0:
-        f['multipart'] = True
-    else:
-        f['multipart'] = False
-    msg_id_in = 'VL_API_' + f['name'].upper()
-    function_print(f['name'], msg_id_in, f['args'], f['pack'], f['multipart'], f['zeroarray'])
-
-
-print "api_func_table = [0] * 10000"
-for f in func_list:
-    #    if f['name'].find('_reply') > 0 or f['name'].find('_details') > 0:
-    msg_id_in = 'VL_API_' + f['name'].upper()
-    api_table_print(f['name'], msg_id_in)
diff --git a/vpp-api/python/pneum/pneum.c b/vpp-api/python/pneum/pneum.c
index ebe47b2f419..3b5455304e5 100644
--- a/vpp-api/python/pneum/pneum.c
+++ b/vpp-api/python/pneum/pneum.c
@@ -52,7 +52,7 @@ typedef struct {
 
 pneum_main_t pneum_main;
 
-extern int wrap_pneum_callback(char *data, int len);
+pneum_callback_t pneum_callback;
 
 /*
  * Satisfy external references when -lvlib is not available.
@@ -62,24 +62,16 @@ void vlib_cli_output (struct vlib_main_t * vm, char * fmt, ...)
   clib_warning ("vlib_cli_output called...");
 }
 
-#define vl_api_version(n,v) static u32 vpe_api_version = v;
-#include <vpp-api/vpe.api.h>
-#undef vl_api_version
 void
-vl_client_add_api_signatures (vl_api_memclnt_create_t *mp)
+pneum_free (void * msg)
 {
-  /*
-   * Send the main API signature in slot 0. This bit of code must
-   * match the checks in ../vpe/api/api.c: vl_msg_api_version_check().
-   */
-  mp->api_versions[0] = clib_host_to_net_u32 (vpe_api_version);
+  vl_msg_api_free (msg);
 }
 
 static void
 pneum_api_handler (void *msg)
 {
   u16 id = ntohs(*((u16 *)msg));
-
   if (id == VL_API_RX_THREAD_EXIT) {
     pneum_main_t *pm = &pneum_main;
     vl_msg_api_free(msg);
@@ -91,8 +83,9 @@ pneum_api_handler (void *msg)
     clib_warning("Message ID %d has wrong length: %d\n", id, l);
 
   /* Call Python callback */
-  (void)wrap_pneum_callback(msg, l);
-  vl_msg_api_free(msg);
+  ASSERT(pneum_callback);
+  (pneum_callback)(msg, l);
+  pneum_free(msg);
 }
 
 static void *
@@ -115,8 +108,22 @@ pneum_rx_thread_fn (void *arg)
   pthread_exit(0);
 }
 
+uword *
+pneum_msg_table_get_hash (void)
+{
+  api_main_t *am = &api_main;
+  return (am->msg_index_by_name_and_crc);
+}
+
 int
-pneum_connect (char * name, char * chroot_prefix)
+pneum_msg_table_size(void)
+{
+  api_main_t *am = &api_main;
+  return hash_elts(am->msg_index_by_name_and_crc);
+}
+
+int
+pneum_connect (char * name, char * chroot_prefix, pneum_callback_t cb)
 {
   int rv = 0;
   pneum_main_t *pm = &pneum_main;
@@ -134,12 +141,15 @@ pneum_connect (char * name, char * chroot_prefix)
     return (-1);
   }
 
-  /* Start the rx queue thread */
-  rv = pthread_create(&pm->rx_thread_handle, NULL, pneum_rx_thread_fn, 0);
-  if (rv) {
-    clib_warning("pthread_create returned %d", rv);
-    vl_client_api_unmap();
-    return (-1);
+  if (cb) {
+    /* Start the rx queue thread */
+    rv = pthread_create(&pm->rx_thread_handle, NULL, pneum_rx_thread_fn, 0);
+    if (rv) {
+      clib_warning("pthread_create returned %d", rv);
+      vl_client_api_unmap();
+      return (-1);
+    }
+    pneum_callback = cb;
   }
 
   pm->connected_to_vlib = 1;
@@ -164,6 +174,7 @@ pneum_disconnect (void)
   if (pm->connected_to_vlib) {
     vl_client_disconnect();
     vl_client_api_unmap();
+    pneum_callback = 0;
   }
   memset (pm, 0, sizeof (*pm));
 
@@ -175,8 +186,11 @@ pneum_read (char **p, int *l)
 {
   unix_shared_memory_queue_t *q;
   api_main_t *am = &api_main;
+  pneum_main_t *pm = &pneum_main;
   uword msg;
 
+  if (!pm->connected_to_vlib) return -1;
+
   *l = 0;
 
   if (am->our_pid == 0) return (-1);
@@ -219,7 +233,9 @@ pneum_write (char *p, int l)
   api_main_t *am = &api_main;
   vl_api_header_t *mp = vl_msg_api_alloc(l);
   unix_shared_memory_queue_t *q;
+  pneum_main_t *pm = &pneum_main;
 
+  if (!pm->connected_to_vlib) return -1;
   if (!mp) return (-1);
   memcpy(mp, p, l);
   mp->client_index = pneum_client_index();
@@ -228,7 +244,7 @@ pneum_write (char *p, int l)
   if (rv != 0) {
     printf("vpe_api_write fails: %d\n", rv);
     /* Clear message */
-    vl_msg_api_free(mp);
+    pneum_free(mp);
   }
   return (rv);
 }
diff --git a/vpp-api/python/pneum/pneum.h b/vpp-api/python/pneum/pneum.h
index 75b10f8df45..00585eb72e5 100644
--- a/vpp-api/python/pneum/pneum.h
+++ b/vpp-api/python/pneum/pneum.h
@@ -15,9 +15,15 @@
 #ifndef included_pneum_h
 #define included_pneum_h
 
-int pneum_connect(char * name, char * chroot_prefix);
+#include <vppinfra/types.h>
+
+typedef void (*pneum_callback_t)(unsigned char * data, int len);
+int pneum_connect(char * name, char * chroot_prefix, pneum_callback_t cb);
 int pneum_disconnect(void);
 int pneum_read(char **data, int *l);
 int pneum_write(char *data, int len);
+void pneum_free(void * msg);
+uword * pneum_msg_table_get_hash (void);
+int pneum_msg_table_size(void);
 
 #endif
diff --git a/vpp-api/python/pneum/test_pneum.c b/vpp-api/python/pneum/test_pneum.c
index 20c29a7e754..49107197f88 100644
--- a/vpp-api/python/pneum/test_pneum.c
+++ b/vpp-api/python/pneum/test_pneum.c
@@ -76,7 +76,7 @@ int main (int argc, char ** argv)
   vl_api_show_version_t message;
   vl_api_show_version_t *mp;
   int async = 1;
-  int rv = pneum_connect("pneum_client", NULL);
+  int rv = pneum_connect("pneum_client", NULL, NULL);
 
   if (rv != 0) {
     printf("Connect failed: %d\n", rv);
diff --git a/vpp-api/python/vpp_papi/pneum_wrap.c b/vpp-api/python/vpp_papi/pneum_wrap.c
index 18c4f233869..5763707b517 100644
--- a/vpp-api/python/vpp_papi/pneum_wrap.c
+++ b/vpp-api/python/vpp_papi/pneum_wrap.c
@@ -15,11 +15,12 @@
 
 #include <Python.h>
 #include "../pneum/pneum.h"
+#include <vppinfra/hash.h>
 
 static PyObject *pneum_callback = NULL;
 
-int
-wrap_pneum_callback (char *data, int len)
+static void
+wrap_pneum_callback (unsigned char * data, int len)
 {
   PyGILState_STATE gstate;
   PyObject *result;//, *arglist;
@@ -38,7 +39,6 @@ wrap_pneum_callback (char *data, int len)
     PyErr_Print();
 
   PyGILState_Release(gstate);
-  return (0);
 }
 
 static PyObject *
@@ -46,22 +46,28 @@ wrap_connect (PyObject *self, PyObject *args)
 {
   char * name, * chroot_prefix = NULL;
   int rv;
-  PyObject * temp;
+  PyObject * temp = NULL;
+  pneum_callback_t cb = NULL;
 
-  if (!PyArg_ParseTuple(args, "sO|s:wrap_connect", &name, &temp, &chroot_prefix))
+  if (!PyArg_ParseTuple(args, "s|Os:wrap_connect",
+			&name, &temp, &chroot_prefix))
     return (NULL);
 
-  if (!PyCallable_Check(temp)) {
-    PyErr_SetString(PyExc_TypeError, "parameter must be callable");
-    return NULL;
-  }
-
-  Py_XINCREF(temp);         /* Add a reference to new callback */
-  Py_XDECREF(pneum_callback);  /* Dispose of previous callback */
-  pneum_callback = temp;       /* Remember new callback */
-
+  if (temp)
+    {
+      if (!PyCallable_Check(temp))
+	{
+	  PyErr_SetString(PyExc_TypeError, "parameter must be callable");
+	  return NULL;
+	}
+
+      Py_XINCREF(temp);         /* Add a reference to new callback */
+      Py_XDECREF(pneum_callback);  /* Dispose of previous callback */
+      pneum_callback = temp;       /* Remember new callback */
+      cb = wrap_pneum_callback;
+    }
   Py_BEGIN_ALLOW_THREADS
-  rv = pneum_connect(name, chroot_prefix);
+    rv = pneum_connect(name, chroot_prefix, cb);
   Py_END_ALLOW_THREADS
   return PyLong_FromLong(rv);
 }
@@ -90,8 +96,6 @@ wrap_write (PyObject *self, PyObject *args)
   return PyLong_FromLong(rv);
 }
 
-void vl_msg_api_free(void *);
-
 static PyObject *
 wrap_read (PyObject *self, PyObject *args)
 {
@@ -110,15 +114,44 @@ wrap_read (PyObject *self, PyObject *args)
 #endif
   if (!ret) { Py_RETURN_NONE; }
 
-  vl_msg_api_free(data);
+  pneum_free(data);
   return ret;
 }
 
+static PyObject *
+wrap_msg_table (PyObject *self, PyObject *args)
+{
+  int i = 0, rv = 0;
+  hash_pair_t *hp;
+  uword *h = pneum_msg_table_get_hash();
+  PyObject *ret = PyList_New(pneum_msg_table_size());
+  if (!ret) goto error;
+  hash_foreach_pair (hp, h,
+  ({
+    PyObject *item = PyTuple_New(2);
+    if (!item) goto error;
+    rv = PyTuple_SetItem(item, 0, PyLong_FromLong((u32)hp->value[0]));
+    if (rv) goto error;
+    rv = PyTuple_SetItem(item, 1, PyString_FromString((char *)hp->key));
+    if (rv) goto error;
+    PyList_SetItem(ret, i, item);
+    i++;
+  }));
+
+  return ret;
+
+ error:
+  /* TODO: Raise exception */
+  printf("msg_table failed");
+  Py_RETURN_NONE;
+}
+
 static PyMethodDef vpp_api_Methods[] = {
   {"connect", wrap_connect, METH_VARARGS, "Connect to the VPP API."},
   {"disconnect", wrap_disconnect, METH_VARARGS, "Disconnect from the VPP API."},
   {"write", wrap_write, METH_VARARGS, "Write data to the VPP API."},
   {"read", wrap_read, METH_VARARGS, "Read data from the VPP API."},
+  {"msg_table", wrap_msg_table, METH_VARARGS, "Get API dictionary."},
   {NULL, NULL, 0, NULL}        /* Sentinel */
 };