Repair vlib API socket server

- Teach vpp_api_test to send/receive API messages over sockets
- Add memfd-based shared memory
- Add api messages to create memfd-based shared memory segments
- vpp_api_test supports both socket and shared memory segment connections
- vpp_api_test pivot from socket to shared memory API messaging
- add socket client support to libvlibclient.so
- dead client reaper sends ping messages, container-friendly
- dead client reaper falls back to kill (<pid>, 0) live checking
  if e.g. a python app goes silent for tens of seconds
- handle ping messages in python client support code
- teach show api ring about pairwise shared-memory segments
- fix ip probing of already resolved destinations (VPP-998)

We'll need this work to implement proper host-stack client isolation

Change-Id: Ic23b65f75c854d0393d9a2e9d6b122a9551be769
Signed-off-by: Dave Barach <dave@barachs.net>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
Signed-off-by: Florin Coras <fcoras@cisco.com>

3 files changed, 167 insertions, 98 deletions

diff --git a/src/vlibapi/api_common.h b/src/vlibapi/api_common.h
index 8e6ab0ff094..324b260ecc6 100644
--- a/src/vlibapi/api_common.h
+++ b/src/vlibapi/api_common.h
@@ -50,14 +50,22 @@ typedef struct vl_api_registration_
 
   u8 *name;			/**< Client name */
 
+  /* Zombie apocalypse checking */
+  f64 last_heard;
+  int last_queue_head;
+  int unanswered_pings;
+
   /** shared memory only: pointer to client input queue */
   unix_shared_memory_queue_t *vl_input_queue;
+  svm_region_t *vlib_rp;
+  void *shmem_hdr;
 
   /* socket server and client */
   u32 clib_file_index;		/**< Socket only: file index */
   i8 *unprocessed_input;	/**< Socket only: pending input */
   u32 unprocessed_msg_length;	/**< Socket only: unprocssed length */
   u8 *output_vector;		/**< Socket only: output vecto  */
+  int *additional_fds_to_close;
 
   /* socket client only */
   u32 server_handle;		/**< Socket client only: server handle */
@@ -235,6 +243,7 @@ typedef struct
   svm_region_t *vlib_rp;
 
   /** Vector of all mapped shared-VM segments */
+  svm_region_t **vlib_private_rps;
   svm_region_t **mapped_shmem_regions;
 
   /** Binary API shared-memory segment header pointer */
diff --git a/src/vlibapi/api_helper_macros.h b/src/vlibapi/api_helper_macros.h
index 052cc6e78b5..fc9374f8a26 100644
--- a/src/vlibapi/api_helper_macros.h
+++ b/src/vlibapi/api_helper_macros.h
@@ -27,82 +27,83 @@
 #define REPLY_MSG_ID_BASE 0
 #endif
 
-#define REPLY_MACRO(t)                                          \
-do {                                                            \
-    unix_shared_memory_queue_t * q;                             \
-    rv = vl_msg_api_pd_handler (mp, rv);                        \
-    q = vl_api_client_index_to_input_queue (mp->client_index);  \
-    if (!q)                                                     \
-        return;                                                 \
-                                                                \
-    rmp = vl_msg_api_alloc (sizeof (*rmp));                     \
-    rmp->_vl_msg_id = htons((t)+(REPLY_MSG_ID_BASE));           \
-    rmp->context = mp->context;                                 \
-    rmp->retval = ntohl(rv);                                    \
-                                                                \
-    vl_msg_api_send_shmem (q, (u8 *)&rmp);                      \
+#define REPLY_MACRO(t)                                                  \
+do {                                                                    \
+    vl_api_registration_t *rp;                                          \
+    rv = vl_msg_api_pd_handler (mp, rv);                                \
+    rp = vl_api_client_index_to_registration (mp->client_index);        \
+    if (rp == 0)                                                        \
+      return;                                                           \
+                                                                        \
+    rmp = vl_msg_api_alloc (sizeof (*rmp));                             \
+    rmp->_vl_msg_id = htons((t)+(REPLY_MSG_ID_BASE));                   \
+    rmp->context = mp->context;                                         \
+    rmp->retval = ntohl(rv);                                            \
+                                                                        \
+    vl_msg_api_send (rp, (u8 *)rmp);                                    \
 } while(0);
 
-#define REPLY_MACRO2(t, body)                                   \
-do {                                                            \
-    unix_shared_memory_queue_t * q;                             \
-    rv = vl_msg_api_pd_handler (mp, rv);                        \
-    q = vl_api_client_index_to_input_queue (mp->client_index);  \
-    if (!q)                                                     \
-        return;                                                 \
-                                                                \
-    rmp = vl_msg_api_alloc (sizeof (*rmp));                     \
-    rmp->_vl_msg_id = htons((t)+(REPLY_MSG_ID_BASE));           \
-    rmp->context = mp->context;                                 \
-    rmp->retval = ntohl(rv);                                    \
-    do {body;} while (0);                                       \
-    vl_msg_api_send_shmem (q, (u8 *)&rmp);                      \
+#define REPLY_MACRO2(t, body)                                           \
+do {                                                                    \
+    vl_api_registration_t *rp;                                          \
+    rv = vl_msg_api_pd_handler (mp, rv);                                \
+    rp = vl_api_client_index_to_registration (mp->client_index);        \
+    if (rp == 0)                                                        \
+      return;                                                           \
+                                                                        \
+    rmp = vl_msg_api_alloc (sizeof (*rmp));                             \
+    rmp->_vl_msg_id = htons((t)+(REPLY_MSG_ID_BASE));                   \
+    rmp->context = mp->context;                                         \
+    rmp->retval = ntohl(rv);                                            \
+    do {body;} while (0);                                               \
+    vl_msg_api_send (rp, (u8 *)rmp);                                    \
 } while(0);
 
-#define REPLY_MACRO3(t, n, body)				\
-do {                                                            \
-    unix_shared_memory_queue_t * q;                             \
-    rv = vl_msg_api_pd_handler (mp, rv);                        \
-    q = vl_api_client_index_to_input_queue (mp->client_index);  \
-    if (!q)                                                     \
-        return;                                                 \
-                                                                \
-    rmp = vl_msg_api_alloc (sizeof (*rmp) + n);                 \
-    rmp->_vl_msg_id = htons((t)+(REPLY_MSG_ID_BASE));           \
-    rmp->context = mp->context;                                 \
-    rmp->retval = ntohl(rv);                                    \
-    do {body;} while (0);                                       \
-    vl_msg_api_send_shmem (q, (u8 *)&rmp);                      \
+#define REPLY_MACRO3(t, n, body)                                        \
+do {                                                                    \
+    vl_api_registration_t *rp;                                          \
+    rv = vl_msg_api_pd_handler (mp, rv);                                \
+    rp = vl_api_client_index_to_registration (mp->client_index);        \
+    if (rp == 0)                                                        \
+      return;                                                           \
+                                                                        \
+    rmp = vl_msg_api_alloc (sizeof (*rmp) + n);                         \
+    rmp->_vl_msg_id = htons((t)+(REPLY_MSG_ID_BASE));                   \
+    rmp->context = mp->context;                                         \
+    rmp->retval = ntohl(rv);                                            \
+    do {body;} while (0);                                               \
+    vl_msg_api_send (rp, (u8 *)rmp);                                    \
 } while(0);
 
-#define REPLY_MACRO4(t, n, body)                                \
-do {                                                            \
-    unix_shared_memory_queue_t * q;                             \
-    u8 is_error = 0;                                            \
-    rv = vl_msg_api_pd_handler (mp, rv);                        \
-    q = vl_api_client_index_to_input_queue (mp->client_index);  \
-    if (!q)                                                     \
-        return;                                                 \
-                                                                \
-    rmp = vl_msg_api_alloc_or_null (sizeof (*rmp) + n);         \
-    if (!rmp)                                                   \
-      {                                                         \
-        /* if there isn't enough memory, try to allocate */     \
-        /* some at least for returning an error */              \
-        rmp = vl_msg_api_alloc (sizeof (*rmp));                 \
-        if (!rmp)                                               \
-          return;                                               \
-                                                                \
-        memset (rmp, 0, sizeof (*rmp));                         \
-        rv = VNET_API_ERROR_TABLE_TOO_BIG;                      \
-        is_error = 1;                                           \
-      }                                                         \
-    rmp->_vl_msg_id = htons((t)+(REPLY_MSG_ID_BASE));           \
-    rmp->context = mp->context;                                 \
-    rmp->retval = ntohl(rv);                                    \
-    if (!is_error)                                              \
-      do {body;} while (0);                                     \
-    vl_msg_api_send_shmem (q, (u8 *)&rmp);                      \
+#define REPLY_MACRO4(t, n, body)                                        \
+do {                                                                    \
+    vl_api_registration_t *rp;                                          \
+    u8 is_error = 0;                                                    \
+    rv = vl_msg_api_pd_handler (mp, rv);                                \
+                                                                        \
+    rp = vl_api_client_index_to_registration (mp->client_index);        \
+    if (rp == 0)                                                        \
+      return;                                                           \
+                                                                        \
+    rmp = vl_msg_api_alloc_or_null (sizeof (*rmp) + n);                 \
+    if (!rmp)                                                           \
+      {                                                                 \
+        /* if there isn't enough memory, try to allocate */             \
+        /* some at least for returning an error */                      \
+        rmp = vl_msg_api_alloc (sizeof (*rmp));                         \
+        if (!rmp)                                                       \
+          return;                                                       \
+                                                                        \
+        memset (rmp, 0, sizeof (*rmp));                                 \
+        rv = VNET_API_ERROR_TABLE_TOO_BIG;                              \
+        is_error = 1;                                                   \
+      }                                                                 \
+    rmp->_vl_msg_id = htons((t)+(REPLY_MSG_ID_BASE));                   \
+    rmp->context = mp->context;                                         \
+    rmp->retval = ntohl(rv);                                            \
+    if (!is_error)                                                      \
+      do {body;} while (0);                                             \
+    vl_msg_api_send (rp, (u8 *)rmp);                                    \
 } while(0);
 
 /* "trust, but verify" */
diff --git a/src/vlibapi/vat_helper_macros.h b/src/vlibapi/vat_helper_macros.h
index 57ad520be1a..5e7f1083947 100644
--- a/src/vlibapi/vat_helper_macros.h
+++ b/src/vlibapi/vat_helper_macros.h
@@ -22,54 +22,113 @@
 /* M: construct, but don't yet send a message */
 #define M(T, mp)                                                \
 do {                                                            \
+    socket_client_main_t *scm = &vam->socket_client_main;	\
     vam->result_ready = 0;                                      \
-    mp = vl_msg_api_alloc_as_if_client(sizeof(*mp));            \
+    if (scm->socket_enable)                                     \
+      {                                                         \
+        mp = (void *)scm->socket_tx_buffer;                     \
+        scm->socket_tx_nbytes = sizeof (*mp);                   \
+      }                                                         \
+    else                                                        \
+      mp = vl_msg_api_alloc_as_if_client(sizeof(*mp));          \
     memset (mp, 0, sizeof (*mp));                               \
     mp->_vl_msg_id = ntohs (VL_API_##T+__plugin_msg_base);      \
     mp->client_index = vam->my_client_index;                    \
 } while(0);
 
+/* MPING: construct a control-ping message, don't send it yet */
+#define MPING(T, mp)                                            \
+do {                                                            \
+    socket_client_main_t *scm = &vam->socket_client_main;	\
+    vam->result_ready = 0;                                      \
+    if (scm->socket_enable)                                     \
+      {                                                         \
+        mp = (void *)scm->socket_tx_buffer;                     \
+        scm->socket_tx_nbytes = sizeof (*mp);                   \
+      }                                                         \
+    else                                                        \
+      mp = vl_msg_api_alloc_as_if_client(sizeof(*mp));          \
+    memset (mp, 0, sizeof (*mp));                               \
+    mp->_vl_msg_id = ntohs (VL_API_##T+__plugin_msg_base);      \
+    mp->client_index = vam->my_client_index;                    \
+    scm->control_pings_outstanding++;                           \
+} while(0);
+
 #define M2(T, mp, n)                                            \
 do {                                                            \
+    socket_client_main_t *scm = &vam->socket_client_main;	\
     vam->result_ready = 0;                                      \
-    mp = vl_msg_api_alloc_as_if_client(sizeof(*mp)+(n));        \
+    if (scm->socket_enable)                                     \
+      {                                                         \
+        mp = (void *)scm->socket_tx_buffer;                     \
+        scm->socket_tx_nbytes = sizeof (*mp) + n;               \
+      }                                                         \
+    else                                                        \
+      mp = vl_msg_api_alloc_as_if_client(sizeof(*mp) + n);      \
     memset (mp, 0, sizeof (*mp));                               \
     mp->_vl_msg_id = ntohs (VL_API_##T+__plugin_msg_base);      \
     mp->client_index = vam->my_client_index;                    \
 } while(0);
 
 /* S: send a message */
-#define S(mp) (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp))
+#define S(mp)                                                   \
+do {                                                            \
+  int n;                                                        \
+  socket_client_main_t *scm = &vam->socket_client_main;         \
+  if (scm->socket_enable)                                       \
+    {                                                           \
+      msgbuf_t msgbuf;                                          \
+                                                                \
+      msgbuf.q = 0;                                             \
+      msgbuf.gc_mark_timestamp = 0;                             \
+      msgbuf.data_len = ntohl(scm->socket_tx_nbytes);           \
+                                                                \
+      n = write (scm->socket_fd, &msgbuf, sizeof (msgbuf));     \
+      if (n < sizeof (msgbuf))                                  \
+        clib_unix_warning ("socket write (msgbuf)");            \
+                                                                \
+      n = write (scm->socket_fd, scm->socket_tx_buffer,         \
+                   scm->socket_tx_nbytes);                      \
+      if (n < scm->socket_tx_nbytes)                            \
+        clib_unix_warning ("socket write (msg)");               \
+    }                                                           \
+  else                                                          \
+    vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp);     \
+ } while (0);
 
 /* W: wait for results, with timeout */
-#define W(ret)					\
-do {                                            \
-    f64 timeout = vat_time_now (vam) + 1.0;     \
-    ret = -99;                                  \
-                                                \
-    while (vat_time_now (vam) < timeout) {      \
-        if (vam->result_ready == 1) {           \
-            ret = vam->retval;                  \
-            break;                              \
-        }                                       \
-        vat_suspend (vam->vlib_main, 1e-5);     \
-    }                                           \
+#define W(ret)                                                  \
+do {                                                            \
+    f64 timeout = vat_time_now (vam) + 1.0;                     \
+    socket_client_main_t *scm = &vam->socket_client_main;	\
+    ret = -99;                                                  \
+                                                                \
+    vl_socket_client_read_reply (scm);                          \
+    while (vat_time_now (vam) < timeout) {                      \
+        if (vam->result_ready == 1) {                           \
+            ret = vam->retval;                                  \
+            break;                                              \
+        }                                                       \
+        vat_suspend (vam->vlib_main, 1e-5);                     \
+    }                                                           \
 } while(0);
 
 /* W2: wait for results, with timeout */
-#define W2(ret, body)				\
-do {                                            \
-    f64 timeout = vat_time_now (vam) + 1.0;     \
-    ret = -99;                                  \
-                                                \
-    while (vat_time_now (vam) < timeout) {      \
-        if (vam->result_ready == 1) {           \
-	  (body);                               \
-	  ret = vam->retval;                    \
-          break;                                \
-        }                                       \
-        vat_suspend (vam->vlib_main, 1e-5);     \
-    }                                           \
+#define W2(ret, body)                                           \
+do {                                                            \
+    f64 timeout = vat_time_now (vam) + 1.0;                     \
+    socket_client_main_t *scm = &vam->socket_client_main;	\
+    ret = -99;                                                  \
+                                                                \
+    vl_socket_client_read_reply (scm);                          \
+    while (vat_time_now (vam) < timeout) {                      \
+        if (vam->result_ready == 1) {                           \
+	  (body);                                               \
+	  ret = vam->retval;                                    \
+          break;                                                \
+        }                                                       \
+        vat_suspend (vam->vlib_main, 1e-5);                     \
+    }                                                           \
 } while(0);