Add client-side msg_name_and_crc -> msg_index table

vppapigen now generates per-message crcs. Verified that whitespace
and real changes in message A don't change the crc for message B, etc.

Fixed the sample and flowperpkt plugins to participate. Others need
the same treatment. They don't build due to python/java language binding
build issues.

To use the scheme:

Client connects as usual.

Then call: u32 vl_api_get_msg_index(char * name_and_crc)
name_and_crc is a string like: "flowperpkt_tx_interface_add_del_753301f3",
aka the message name with _%08x <expected crc> appended.

Try these vpp-api-test commands to play with it:

vat# dump_msg_api_table
     <snip>
 [366]: punt_reply_cca27fbe
 [367]: ipsec_spd_dump_5e9ae88e
 [368]: ipsec_spd_details_6f7821b0
 [369]: sample_macswap_enable_disable_0f2813e2
 [370]: sample_macswap_enable_disable_reply_476738e5
 [371]: flowperpkt_tx_interface_add_del_753301f3
 [372]: flowperpkt_tx_interface_add_del_reply_d47e6e0b

vat# get_msg_id sample_macswap_enable_disable_reply_476738e5
 'sample_macswap_enable_disable_reply_476738e5' has message index 370

vat# get_msg_id sample_macswap_enable_disable_reply_476738e3
 'sample_macswap_enable_disable_reply_476738e3' not found

CRCs may vary, etc.

vppapigen is used to build a set of JSON representations
of each API file from vpp-api/Makefile.am and that is in
turn used by each language binding (Java, Python, Lua).

Change-Id: I3d64582e779dac5f20cddec79c562c288d8fd9c6
Signed-off-by: Dave Barach <dave@barachs.net>
Signed-off-by: Ole Troan <ot@cisco.com>

5 files changed, 115 insertions, 30 deletions

diff --git a/vlib-api/vlibapi/api.h b/vlib-api/vlibapi/api.h
index a932bf42177..10ca443b434 100644
--- a/vlib-api/vlibapi/api.h
+++ b/vlib-api/vlibapi/api.h
@@ -87,11 +87,14 @@ typedef struct
   u8 **traces;
 } vl_api_trace_t;
 
-typedef CLIB_PACKED (struct
-		     {
-		     u8 endian; u8 wrapped;
-		     u32 nitems;
-		     }) vl_api_trace_file_header_t;
+/* *INDENT-OFF* */
+typedef CLIB_PACKED
+(struct
+ {
+   u8 endian; u8 wrapped;
+   u32 nitems;
+}) vl_api_trace_file_header_t;
+/* *INDENT-ON* */
 
 typedef enum
 {
@@ -132,6 +135,8 @@ typedef struct
   struct vl_shmem_hdr_ *shmem_hdr;
   vl_api_registration_t **vl_clients;
 
+  u8 *serialized_message_table_in_shmem;
+
   /* For plugin msg allocator */
   u16 first_available_msg_id;
 
@@ -178,6 +183,9 @@ typedef struct
 
   i32 vlib_signal;
 
+  /* client side message index hash table */
+  uword *msg_index_by_name_and_crc;
+
   char *region_name;
   char *root_path;
 } api_main_t;
@@ -188,6 +196,7 @@ typedef struct
 {
   int id;
   char *name;
+  u32 crc;
   void *handler;
   void *cleanup;
   void *endian;
@@ -242,6 +251,8 @@ int vl_msg_api_pd_handler (void *mp, int rv);
 
 void vl_msg_api_set_first_available_msg_id (u16 first_avail);
 u16 vl_msg_api_get_msg_ids (char *name, int n);
+void vl_msg_api_add_msg_name_crc (api_main_t * am, char *string, u32 id);
+u32 vl_api_get_msg_index (u8 * name_and_crc);
 
 /* node_serialize.c prototypes */
 u8 *vlib_node_serialize (vlib_node_main_t * nm, u8 * vector,
diff --git a/vlib-api/vlibapi/api_shared.c b/vlib-api/vlibapi/api_shared.c
index 511f693849d..2b2d81c2735 100644
--- a/vlib-api/vlibapi/api_shared.c
+++ b/vlib-api/vlibapi/api_shared.c
@@ -1312,6 +1312,25 @@ vl_msg_api_get_msg_ids (char *name, int n)
   return rv;
 }
 
+void
+vl_msg_api_add_msg_name_crc (api_main_t * am, char *string, u32 id)
+{
+  uword *p;
+
+  if (am->msg_index_by_name_and_crc == 0)
+    am->msg_index_by_name_and_crc = hash_create_string (0, sizeof (uword));
+
+  p = hash_get_mem (am->msg_index_by_name_and_crc, string);
+  if (p)
+    {
+      clib_warning ("attempt to redefine '%s' ignored...", string);
+      return;
+    }
+
+  hash_set_mem (am->msg_index_by_name_and_crc, string, id);
+}
+
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/vlib-api/vlibmemory/memclnt.api b/vlib-api/vlibmemory/memclnt.api
index c2758238a96..2f654caf2c1 100644
--- a/vlib-api/vlibmemory/memclnt.api
+++ b/vlib-api/vlibmemory/memclnt.api
@@ -28,9 +28,10 @@ define memclnt_create {
 
 define memclnt_create_reply {
     i32 response;               /* Non-negative = success */
-    u64 handle;               /* handle by which vlib knows this client */
+    u64 handle;                 /* handle by which vlib knows this client */
     u32 index;                  /* index, used e.g. by API trace replay */
     u32 context;                /* opaque value from the create request */
+    u64 message_table;          /* serialized message table in shmem */
 };
 
 /*
diff --git a/vlib-api/vlibmemory/memory_shared.c b/vlib-api/vlibmemory/memory_shared.c
index 134fcd52c84..6adc69fc05f 100644
--- a/vlib-api/vlibmemory/memory_shared.c
+++ b/vlib-api/vlibmemory/memory_shared.c
@@ -541,28 +541,62 @@ vl_msg_api_send_shmem_nolock (unix_shared_memory_queue_t * q, u8 * elem)
 static void
 vl_api_memclnt_create_reply_t_handler (vl_api_memclnt_create_reply_t * mp)
 {
+  serialize_main_t _sm, *sm = &_sm;
   api_main_t *am = &api_main;
-  int rv;
+  u8 *tblv;
+  u32 nmsgs;
+  int i;
+  u8 *name_and_crc;
+  u32 msg_index;
 
   am->my_client_index = mp->index;
   am->my_registration = (vl_api_registration_t *) (uword) mp->handle;
 
-  rv = ntohl (mp->response);
+  /* Clean out any previous hash table (unlikely) */
+  if (am->msg_index_by_name_and_crc)
+    {
+      int i;
+      u8 **keys = 0;
+      hash_pair_t *hp;
+      /* *INDENT-OFF* */
+      hash_foreach_pair (hp, am->msg_index_by_name_and_crc,
+      ({
+        vec_add1 (keys, (u8 *) hp->key);
+      }));
+      /* *INDENT-ON* */
+      for (i = 0; i < vec_len (keys); i++)
+	vec_free (keys[i]);
+      vec_free (keys);
+    }
 
-  if (rv < 0)
-    clib_warning ("WARNING: API mismatch detected");
-}
+  am->msg_index_by_name_and_crc = hash_create_string (0, sizeof (uword));
 
-void vl_client_add_api_signatures (vl_api_memclnt_create_t * mp)
-  __attribute__ ((weak));
+  /* Recreate the vnet-side API message handler table */
+  tblv = (u8 *) mp->message_table;
+  serialize_open_vector (sm, tblv);
+  unserialize_integer (sm, &nmsgs, sizeof (u32));
 
-void
-vl_client_add_api_signatures (vl_api_memclnt_create_t * mp)
+  for (i = 0; i < nmsgs; i++)
+    {
+      msg_index = unserialize_likely_small_unsigned_integer (sm);
+      unserialize_cstring (sm, (char **) &name_and_crc);
+      hash_set_mem (am->msg_index_by_name_and_crc, name_and_crc, msg_index);
+    }
+}
+
+u32
+vl_api_get_msg_index (u8 * name_and_crc)
 {
-  int i;
+  api_main_t *am = &api_main;
+  uword *p;
 
-  for (i = 0; i < ARRAY_LEN (mp->api_versions); i++)
-    mp->api_versions[i] = 0;
+  if (am->msg_index_by_name_and_crc)
+    {
+      p = hash_get_mem (am->msg_index_by_name_and_crc, name_and_crc);
+      if (p)
+	return p[0];
+    }
+  return ~0;
 }
 
 int
@@ -617,8 +651,6 @@ vl_client_connect (char *name, int ctx_quota, int input_queue_size)
   mp->input_queue = (uword) vl_input_queue;
   strncpy ((char *) mp->name, name, sizeof (mp->name) - 1);
 
-  vl_client_add_api_signatures (mp);
-
   vl_msg_api_send_shmem (shmem_hdr->vl_input_queue, (u8 *) & mp);
 
   while (1)
diff --git a/vlib-api/vlibmemory/memory_vlib.c b/vlib-api/vlibmemory/memory_vlib.c
index ca3eb141f7e..5f97f1611d1 100644
--- a/vlib-api/vlibmemory/memory_vlib.c
+++ b/vlib-api/vlibmemory/memory_vlib.c
@@ -100,13 +100,28 @@ vl_msg_api_send (vl_api_registration_t * rp, u8 * elem)
     }
 }
 
-int vl_msg_api_version_check (vl_api_memclnt_create_t * mp)
-  __attribute__ ((weak));
-
-int
-vl_msg_api_version_check (vl_api_memclnt_create_t * mp)
+u8 *
+vl_api_serialize_message_table (api_main_t * am, u8 * vector)
 {
-  return 0;
+  serialize_main_t _sm, *sm = &_sm;
+  hash_pair_t *hp;
+  u32 nmsg = hash_elts (am->msg_index_by_name_and_crc);
+
+  serialize_open_vector (sm, vector);
+
+  /* serialize the count */
+  serialize_integer (sm, nmsg, sizeof (u32));
+
+  hash_foreach_pair (hp, am->msg_index_by_name_and_crc, (
+							  {
+							  serialize_likely_small_unsigned_integer
+							  (sm, hp->value[0]);
+							  serialize_cstring
+							  (sm,
+							   (char *) hp->key);
+							  }));
+
+  return serialize_close_vector (sm);
 }
 
 /*
@@ -120,12 +135,10 @@ vl_api_memclnt_create_t_handler (vl_api_memclnt_create_t * mp)
   vl_api_memclnt_create_reply_t *rp;
   svm_region_t *svm;
   unix_shared_memory_queue_t *q;
-  int rv;
+  int rv = 0;
   void *oldheap;
   api_main_t *am = &api_main;
-
-  /* Indicate API version mismatch if appropriate */
-  rv = vl_msg_api_version_check (mp);
+  u8 *serialized_message_table = 0;
 
   /*
    * This is tortured. Maintain a vlib-address-space private
@@ -157,6 +170,9 @@ vl_api_memclnt_create_t_handler (vl_api_memclnt_create_t * mp)
 
   svm = am->vlib_rp;
 
+  if (am->serialized_message_table_in_shmem == 0)
+    serialized_message_table = vl_api_serialize_message_table (am, 0);
+
   pthread_mutex_lock (&svm->mutex);
   oldheap = svm_push_data_heap (svm);
   *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
@@ -171,10 +187,15 @@ vl_api_memclnt_create_t_handler (vl_api_memclnt_create_t * mp)
 
   regp->name = format (0, "%s", mp->name);
   vec_add1 (regp->name, 0);
+  if (serialized_message_table)
+    am->serialized_message_table_in_shmem =
+      vec_dup (serialized_message_table);
 
   pthread_mutex_unlock (&svm->mutex);
   svm_pop_heap (oldheap);
 
+  vec_free (serialized_message_table);
+
   rp = vl_msg_api_alloc (sizeof (*rp));
   rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_REPLY);
   rp->handle = (uword) regp;
@@ -183,6 +204,7 @@ vl_api_memclnt_create_t_handler (vl_api_memclnt_create_t * mp)
      am->shmem_hdr->application_restarts);
   rp->context = mp->context;
   rp->response = ntohl (rv);
+  rp->message_table = (u64) am->serialized_message_table_in_shmem;
 
   vl_msg_api_send_shmem (q, (u8 *) & rp);
 }