api: vppapitrace JSON/API trace converter

usage: vppapitrace.py [-h] [--debug] [--apidir APIDIR] {convert,replay} ...

optional arguments:
  -h, --help        show this help message and exit
  --debug           enable debug mode
  --apidir APIDIR   Location of JSON API definitions

subcommands:
  valid subcommands

  {convert,replay}  additional help
    convert         Convert API trace to JSON or Python and back
    replay          Replay messages to running VPP instance

To convert an API trace file to JSON:
vppapitrace convert /tmp/api.trace trace.json

To convert an (edited) JSON file back to API trace for replay:
vppapitrace convert trace.json api-edited.trace

To generate a Python file that can be replayed:
vppapitrace convert /tmp/api.trace trace.py
vppapitrace convert trace.json trace.py

Replay it to a running VPP instance:
vppapitrace replay --socket /tmp/api.trace

In VPP that file can be replayed with:
vpp# api trace replay api-edited.trace

This patch also modifies the API binary trace format, to include the
message id to message name table.

Change-Id: Ie6441efb53c1c93c9f778f6ae9c1758bccc8dd87
Type: refactor
Signed-off-by: Ole Troan <ot@cisco.com>

3 files changed, 38 insertions, 56 deletions

diff --git a/src/vlibmemory/socket_api.c b/src/vlibmemory/socket_api.c
index 5aad8a9598f..868298ccc85 100644
--- a/src/vlibmemory/socket_api.c
+++ b/src/vlibmemory/socket_api.c
@@ -692,9 +692,9 @@ reply:
 }
 
 #define foreach_vlib_api_msg                    	\
-_(SOCKCLNT_CREATE, sockclnt_create)             	\
-_(SOCKCLNT_DELETE, sockclnt_delete)			\
-_(SOCK_INIT_SHM, sock_init_shm)
+  _(SOCKCLNT_CREATE, sockclnt_create, 1)             	\
+  _(SOCKCLNT_DELETE, sockclnt_delete, 1)		\
+  _(SOCK_INIT_SHM, sock_init_shm, 1)
 
 clib_error_t *
 vl_sock_api_init (vlib_main_t * vm)
@@ -710,13 +710,13 @@ vl_sock_api_init (vlib_main_t * vm)
   if (sm->socket_name == 0)
     return 0;
 
-#define _(N,n)                                                  \
+#define _(N,n,t)						\
     vl_msg_api_set_handlers(VL_API_##N, #n,                     \
                            vl_api_##n##_t_handler,              \
                            vl_noop_handler,                     \
                            vl_api_##n##_t_endian,               \
                            vl_api_##n##_t_print,                \
-                           sizeof(vl_api_##n##_t), 1);
+                           sizeof(vl_api_##n##_t), t);
   foreach_vlib_api_msg;
 #undef _
 
diff --git a/src/vlibmemory/vlib_api.c b/src/vlibmemory/vlib_api.c
index 73dd5bda33c..e1a6bd18d55 100644
--- a/src/vlibmemory/vlib_api.c
+++ b/src/vlibmemory/vlib_api.c
@@ -68,29 +68,6 @@ vl_api_trace_plugin_msg_ids_t_print (vl_api_trace_plugin_msg_ids_t * a,
 #include <vlibmemory/vl_memory_api_h.h>
 #undef vl_endianfun
 
-u8 *
-vl_api_serialize_message_table (api_main_t * am, u8 * vector)
-{
-  serialize_main_t _sm, *sm = &_sm;
-  hash_pair_t *hp;
-  u32 nmsg = hash_elts (am->msg_index_by_name_and_crc);
-
-  serialize_open_vector (sm, vector);
-
-  /* serialize the count */
-  serialize_integer (sm, nmsg, sizeof (u32));
-
-  /* *INDENT-OFF* */
-  hash_foreach_pair (hp, am->msg_index_by_name_and_crc,
-  ({
-    serialize_likely_small_unsigned_integer (sm, hp->value[0]);
-    serialize_cstring (sm, (char *) hp->key);
-  }));
-  /* *INDENT-ON* */
-
-  return serialize_close_vector (sm);
-}
-
 static void
 vl_api_get_first_msg_id_t_handler (vl_api_get_first_msg_id_t * mp)
 {
diff --git a/src/vlibmemory/vlib_api_cli.c b/src/vlibmemory/vlib_api_cli.c
index b5fe151a2c0..0d5ce96fbdf 100755
--- a/src/vlibmemory/vlib_api_cli.c
+++ b/src/vlibmemory/vlib_api_cli.c
@@ -402,10 +402,9 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
   struct stat statb;
   size_t file_size;
   u8 *msg;
-  u8 endian_swap_needed = 0;
   api_main_t *am = &api_main;
   u8 *tmpbuf = 0;
-  u32 nitems;
+  u32 nitems, nitems_msgtbl;
   void **saved_print_handlers = 0;
 
   fd = open ((char *) filename, O_RDONLY);
@@ -443,14 +442,7 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
     }
   close (fd);
 
-  if ((clib_arch_is_little_endian && hp->endian == VL_API_BIG_ENDIAN)
-      || (clib_arch_is_big_endian && hp->endian == VL_API_LITTLE_ENDIAN))
-    endian_swap_needed = 1;
-
-  if (endian_swap_needed)
-    nitems = ntohl (hp->nitems);
-  else
-    nitems = hp->nitems;
+  nitems = ntohl (hp->nitems);
 
   if (last_index == (u32) ~ 0)
     {
@@ -473,9 +465,26 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       saved_print_handlers = (void **) vec_dup (am->msg_print_handlers);
       vl_msg_api_custom_dump_configure (am);
     }
+  msg = (u8 *) (hp + 1);
 
+  u16 *msgid_vec = 0;
+  serialize_main_t _sm, *sm = &_sm;
+  u32 msgtbl_size = ntohl (hp->msgtbl_size);
+  u8 *name_and_crc;
 
-  msg = (u8 *) (hp + 1);
+  unserialize_open_data (sm, msg, msgtbl_size);
+  unserialize_integer (sm, &nitems_msgtbl, sizeof (u32));
+
+  for (i = 0; i < nitems_msgtbl; i++)
+    {
+      u16 msg_index = unserialize_likely_small_unsigned_integer (sm);
+      unserialize_cstring (sm, (char **) &name_and_crc);
+      u16 msg_index2 = vl_msg_api_get_msg_index (name_and_crc);
+      vec_validate (msgid_vec, msg_index);
+      msgid_vec[msg_index] = msg_index2;
+    }
+
+  msg += msgtbl_size;
 
   for (i = 0; i < first_index; i++)
     {
@@ -486,11 +495,9 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       size = clib_host_to_net_u32 (*(u32 *) msg);
       msg += sizeof (u32);
 
-      if (clib_arch_is_little_endian)
-	msg_id = ntohs (*((u16 *) msg));
-      else
-	msg_id = *((u16 *) msg);
-
+      msg_id = ntohs (*((u16 *) msg));
+      if (msg_id < vec_len (msgid_vec))
+	msg_id = msgid_vec[msg_id];
       cfgp = am->api_trace_cfg + msg_id;
       if (!cfgp)
 	{
@@ -507,7 +514,6 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
   for (; i <= last_index; i++)
     {
       trace_cfg_t *cfgp;
-      u16 *msg_idp;
       u16 msg_id;
       int size;
 
@@ -517,10 +523,11 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       size = clib_host_to_net_u32 (*(u32 *) msg);
       msg += sizeof (u32);
 
-      if (clib_arch_is_little_endian)
-	msg_id = ntohs (*((u16 *) msg));
-      else
-	msg_id = *((u16 *) msg);
+      msg_id = ntohs (*((u16 *) msg));
+      if (msg_id < vec_len (msgid_vec))
+	{
+	  msg_id = msgid_vec[msg_id];
+	}
 
       cfgp = am->api_trace_cfg + msg_id;
       if (!cfgp)
@@ -538,12 +545,10 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       clib_memset (tmpbuf, 0xf, sizeof (uword));
 
       /*
-       * Endian swap if needed. All msg data is supposed to be
-       * in network byte order. All msg handlers are supposed to
-       * know that. The generic message dumpers don't know that.
-       * One could fix apigen, I suppose.
+       * Endian swap if needed. All msg data is supposed to be in
+       * network byte order.
        */
-      if ((which == DUMP && clib_arch_is_little_endian) || endian_swap_needed)
+      if ((which == DUMP && clib_arch_is_little_endian))
 	{
 	  void (*endian_fp) (void *);
 	  if (msg_id >= vec_len (am->msg_endian_handlers)
@@ -562,7 +567,7 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       /* msg_id always in network byte order */
       if (clib_arch_is_little_endian)
 	{
-	  msg_idp = (u16 *) (tmpbuf + sizeof (uword));
+	  u16 *msg_idp = (u16 *) (tmpbuf + sizeof (uword));
 	  *msg_idp = msg_id;
 	}
 
@@ -1051,7 +1056,7 @@ dump_api_table_file_command_fn (vlib_main_t * vm,
       item->crc = extract_crc (name_and_crc);
       item->which = 0;		/* file */
     }
-  serialize_close (sm);
+  unserialize_close (sm);
 
   /* Compare with the current image? */
   if (compare_current)