vcl: wait for segments with segment handle

Instead of waiting for notification from binary api.

Change-Id: I5ecab857d6bcdbed62d6bb06709570c4cf6b19ea
Signed-off-by: Florin Coras <fcoras@cisco.com>

10 files changed, 154 insertions, 71 deletions

diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c
index 7f3a5741af3..c6e7fdf369d 100644
--- a/src/vcl/vcl_bapi.c
+++ b/src/vcl/vcl_bapi.c
@@ -62,7 +62,8 @@ static void
 }
 
 static int
-ssvm_segment_attach (char *name, ssvm_segment_type_t type, int fd)
+vcl_segment_attach (u64 segment_handle, char *name, ssvm_segment_type_t type,
+		    int fd)
 {
   svm_fifo_segment_create_args_t _a, *a = &_a;
   int rv;
@@ -79,15 +80,37 @@ ssvm_segment_attach (char *name, ssvm_segment_type_t type, int fd)
       clib_warning ("svm_fifo_segment_attach ('%s') failed", name);
       return rv;
     }
+  vcl_segment_table_add (segment_handle, a->new_segment_indices[0]);
   vec_reset_length (a->new_segment_indices);
   return 0;
 }
 
 static void
+vcl_segment_detach (u64 segment_handle)
+{
+  svm_fifo_segment_private_t *segment;
+  u32 segment_index;
+
+  segment_index = vcl_segment_table_lookup (segment_handle);
+  if (segment_index == (u32) ~ 0)
+    return;
+  segment = svm_fifo_segment_get_segment (segment_index);
+  svm_fifo_segment_delete (segment);
+  vcl_segment_table_del (segment_handle);
+}
+
+static u64
+vcl_vpp_worker_segment_handle (u32 wrk_index)
+{
+  return (VCL_INVALID_SEGMENT_HANDLE - wrk_index - 1);
+}
+
+static void
 vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t *
 					   mp)
 {
   vcl_worker_t *wrk = vcl_worker_get (0);
+  u64 segment_handle;
   u32 n_fds = 0;
   int *fds = 0;
 
@@ -100,19 +123,27 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t *
 
   wrk->app_event_queue = uword_to_pointer (mp->app_event_queue_address,
 					   svm_msg_q_t *);
+  segment_handle = clib_net_to_host_u64 (mp->segment_handle);
+  if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
+    {
+      clib_warning ("invalid segment handle");
+      return;
+    }
+
   if (mp->n_fds)
     {
       vec_validate (fds, mp->n_fds);
       vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5);
 
       if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT)
-	if (ssvm_segment_attach ("vpp-mq-seg", SSVM_SEGMENT_MEMFD,
-				 fds[n_fds++]))
+	if (vcl_segment_attach (vcl_vpp_worker_segment_handle (0),
+				"vpp-mq-seg", SSVM_SEGMENT_MEMFD,
+				fds[n_fds++]))
 	  return;
 
       if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT)
-	if (ssvm_segment_attach ((char *) mp->segment_name,
-				 SSVM_SEGMENT_MEMFD, fds[n_fds++]))
+	if (vcl_segment_attach (segment_handle, (char *) mp->segment_name,
+				SSVM_SEGMENT_MEMFD, fds[n_fds++]))
 	  return;
 
       if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD)
@@ -126,8 +157,8 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t *
     }
   else
     {
-      if (ssvm_segment_attach ((char *) mp->segment_name, SSVM_SEGMENT_SHM,
-			       -1))
+      if (vcl_segment_attach (segment_handle, (char *) mp->segment_name,
+			      SSVM_SEGMENT_SHM, -1))
 	return;
     }
 
@@ -140,6 +171,7 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t *
 					   mp)
 {
   int n_fds = 0, *fds = 0;
+  u64 segment_handle;
   vcl_worker_t *wrk;
   u32 wrk_index;
 
@@ -159,19 +191,27 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t *
   wrk->app_event_queue = uword_to_pointer (mp->app_event_queue_address,
 					   svm_msg_q_t *);
 
+  segment_handle = clib_net_to_host_u64 (mp->segment_handle);
+  if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
+    {
+      clib_warning ("invalid segment handle");
+      return;
+    }
+
   if (mp->n_fds)
     {
       vec_validate (fds, mp->n_fds);
       vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5);
 
       if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT)
-	if (ssvm_segment_attach ("vpp-worker-seg", SSVM_SEGMENT_MEMFD,
-				 fds[n_fds++]))
+	if (vcl_segment_attach
+	    (vcl_vpp_worker_segment_handle (wrk->wrk_index), "vpp-worker-seg",
+	     SSVM_SEGMENT_MEMFD, fds[n_fds++]))
 	  goto failed;
 
       if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT)
-	if (ssvm_segment_attach ((char *) mp->segment_name,
-				 SSVM_SEGMENT_MEMFD, fds[n_fds++]))
+	if (vcl_segment_attach (segment_handle, (char *) mp->segment_name,
+				SSVM_SEGMENT_MEMFD, fds[n_fds++]))
 	  goto failed;
 
       if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD)
@@ -185,8 +225,8 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t *
     }
   else
     {
-      if (ssvm_segment_attach ((char *) mp->segment_name, SSVM_SEGMENT_SHM,
-			       -1))
+      if (vcl_segment_attach (segment_handle, (char *) mp->segment_name,
+			      SSVM_SEGMENT_SHM, -1))
 	goto failed;
     }
   vcm->app_state = STATE_APP_READY;
@@ -212,18 +252,24 @@ static void
 vl_api_map_another_segment_t_handler (vl_api_map_another_segment_t * mp)
 {
   ssvm_segment_type_t seg_type = SSVM_SEGMENT_SHM;
+  u64 segment_handle;
   int fd = -1;
 
-  vcm->mounting_segment = 1;
-
   if (mp->fd_flags)
     {
       vl_socket_client_recv_fd_msg (&fd, 1, 5);
       seg_type = SSVM_SEGMENT_MEMFD;
     }
 
-  if (PREDICT_FALSE (ssvm_segment_attach ((char *) mp->segment_name,
-					  seg_type, fd)))
+  segment_handle = clib_net_to_host_u64 (mp->segment_handle);
+  if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
+    {
+      clib_warning ("invalid segment handle");
+      return;
+    }
+
+  if (vcl_segment_attach (segment_handle, (char *) mp->segment_name,
+			  seg_type, fd))
     {
       clib_warning ("VCL<%d>: svm_fifo_segment_attach ('%s') failed",
 		    getpid (), mp->segment_name);
@@ -232,20 +278,14 @@ vl_api_map_another_segment_t_handler (vl_api_map_another_segment_t * mp)
 
   VDBG (1, "VCL<%d>: mapped new segment '%s' size %d", getpid (),
 	mp->segment_name, mp->segment_size);
-  vcm->mounting_segment = 0;
 }
 
 static void
 vl_api_unmap_segment_t_handler (vl_api_unmap_segment_t * mp)
 {
-
-/*
- * XXX Need segment_name to session_id hash,
- * XXX - have sessionID by handle hash currently
- */
-
-  VDBG (1, "Unmapped segment '%s'",
-	clib_net_to_host_u64 (mp->segment_handle));
+  u64 segment_handle = clib_net_to_host_u64 (mp->segment_handle);
+  vcl_segment_detach (segment_handle);
+  VDBG (1, "Unmapped segment: %d", segment_handle);
 }
 
 static void
diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c
index 86dccfe1cc9..76ae0e7719f 100644
--- a/src/vcl/vcl_private.c
+++ b/src/vcl/vcl_private.c
@@ -442,6 +442,36 @@ vcl_session_get_refcnt (vcl_session_t * s)
   return 0;
 }
 
+void
+vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index)
+{
+  clib_rwlock_writer_lock (&vcm->segment_table_lock);
+  hash_set (vcm->segment_table, segment_handle, svm_segment_index);
+  clib_rwlock_writer_unlock (&vcm->segment_table_lock);
+}
+
+u32
+vcl_segment_table_lookup (u64 segment_handle)
+{
+  uword *seg_indexp;
+
+  clib_rwlock_reader_lock (&vcm->segment_table_lock);
+  seg_indexp = hash_get (vcm->segment_table, segment_handle);
+  clib_rwlock_reader_unlock (&vcm->segment_table_lock);
+
+  if (!seg_indexp)
+    return VCL_INVALID_SEGMENT_INDEX;
+  return ((u32) * seg_indexp);
+}
+
+void
+vcl_segment_table_del (u64 segment_handle)
+{
+  clib_rwlock_writer_lock (&vcm->segment_table_lock);
+  hash_unset (vcm->segment_table, segment_handle);
+  clib_rwlock_writer_unlock (&vcm->segment_table_lock);
+}
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index 0420322d3f4..6ca0471e965 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -309,9 +309,6 @@ typedef struct vppcom_main_t_
   /** VCL configuration */
   vppcom_cfg_t cfg;
 
-  /** Flag indicating that a new segment is being mounted */
-  volatile u32 mounting_segment;
-
   volatile u32 forking;
 
   /** Workers */
@@ -320,7 +317,15 @@ typedef struct vppcom_main_t_
   /** Lock to protect worker registrations */
   clib_spinlock_t workers_lock;
 
+  /** Pool of shared sessions */
   vcl_shared_session_t *shared_sessions;
+
+  /** Lock to protect segment hash table */
+  clib_rwlock_t segment_table_lock;
+
+  /** Mapped segments table */
+  uword *segment_table;
+
 #ifdef VCL_ELOG
   /* VPP Event-logger */
   elog_main_t elog_main;
@@ -335,6 +340,8 @@ typedef struct vppcom_main_t_
 extern vppcom_main_t *vcm;
 
 #define VCL_INVALID_SESSION_INDEX ((u32)~0)
+#define VCL_INVALID_SEGMENT_INDEX ((u32)~0)
+#define VCL_INVALID_SEGMENT_HANDLE ((u64)~0)
 
 static inline vcl_session_t *
 vcl_session_alloc (vcl_worker_t * wrk)
@@ -502,6 +509,10 @@ void vcl_worker_share_sessions (u32 parent_wrk_index);
 int vcl_worker_unshare_session (vcl_worker_t * wrk, vcl_session_t * s);
 int vcl_session_get_refcnt (vcl_session_t * s);
 
+void vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index);
+u32 vcl_segment_table_lookup (u64 segment_handle);
+void vcl_segment_table_del (u64 segment_handle);
+
 static inline vcl_worker_t *
 vcl_worker_get (u32 wrk_index)
 {
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 195e6cbc602..34b663b9831 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -22,46 +22,26 @@
 
 __thread uword __vcl_worker_index = ~0;
 
-static u8 not_ready;
 
-void
-sigsegv_signal (int signum)
+static int
+vcl_wait_for_segment (u64 segment_handle)
 {
-  not_ready = 1;
-}
+  vcl_worker_t *wrk = vcl_worker_get_current ();
+  u32 wait_for_seconds = 10, segment_index;
+  f64 timeout;
 
-static void
-vcl_wait_for_memory (void *mem)
-{
-  u8 __clib_unused test;
-  if (vcm->mounting_segment)
-    {
-      while (vcm->mounting_segment)
-	;
-      return;
-    }
-  if (1 || vcm->debug)
-    {
-      usleep (1e5);
-      return;
-    }
-  if (signal (SIGSEGV, sigsegv_signal))
-    {
-      perror ("signal()");
-      return;
-    }
-  not_ready = 0;
+  if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
+    return 1;
 
-again:
-  test = *(u8 *) mem;
-  if (not_ready)
+  timeout = clib_time_now (&wrk->clib_time) + wait_for_seconds;
+  while (clib_time_now (&wrk->clib_time) < timeout)
     {
-      not_ready = 0;
-      usleep (1);
-      goto again;
+      segment_index = vcl_segment_table_lookup (segment_handle);
+      if (segment_index != VCL_INVALID_SEGMENT_INDEX)
+	return 0;
+      usleep (10);
     }
-
-  signal (SIGSEGV, SIG_DFL);
+  return 1;
 }
 
 const char *
@@ -281,7 +261,12 @@ vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp)
 					     svm_msg_q_t *);
       session->our_evt_q = uword_to_pointer (mp->server_event_queue_address,
 					     svm_msg_q_t *);
-      vcl_wait_for_memory (session->vpp_evt_q);
+      if (vcl_wait_for_segment (mp->segment_handle))
+	{
+	  clib_warning ("segment for session %u couldn't be mounted!",
+			session->session_index);
+	  return VCL_INVALID_SESSION_INDEX;
+	}
       rx_fifo->master_session_index = session->session_index;
       tx_fifo->master_session_index = session->session_index;
       rx_fifo->master_thread_index = vcl_get_worker_index ();
@@ -360,7 +345,13 @@ vcl_session_connected_handler (vcl_worker_t * wrk,
 
   rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *);
   tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *);
-  vcl_wait_for_memory (rx_fifo);
+  if (vcl_wait_for_segment (mp->segment_handle))
+    {
+      clib_warning ("segment for session %u couldn't be mounted!",
+		    session->session_index);
+      return VCL_INVALID_SESSION_INDEX;
+    }
+
   rx_fifo->client_session_index = session_index;
   tx_fifo->client_session_index = session_index;
   rx_fifo->client_thread_index = vcl_get_worker_index ();
@@ -436,8 +427,8 @@ vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp)
   session = vcl_session_get (wrk, sid);
   if (mp->retval)
     {
-      VDBG (0, "VCL<%d>: ERROR: vpp handle 0x%llx, sid %u: bind failed: %U",
-	    getpid (), mp->handle, sid, format_api_error, ntohl (mp->retval));
+      VERR ("vpp handle 0x%llx, sid %u: bind failed: %U", mp->handle, sid,
+	    format_api_error, mp->retval);
       if (session)
 	{
 	  session->session_state = STATE_FAILED;
@@ -780,6 +771,7 @@ vppcom_app_create (char *app_name)
 			      20 /* timeout in secs */ );
   pool_alloc (vcm->workers, vcl_cfg->max_workers);
   clib_spinlock_init (&vcm->workers_lock);
+  clib_rwlock_init (&vcm->segment_table_lock);
   pthread_atfork (NULL, vcl_app_fork_parent_handler,
 		  vcl_app_fork_child_handler);
 
diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c
index 69f33290a73..f3628966ae5 100644
--- a/src/vnet/session/application.c
+++ b/src/vnet/session/application.c
@@ -1591,8 +1591,8 @@ application_local_session_connect (app_worker_t * client_wrk,
 				   local_session_t * ll, u32 opaque)
 {
   u32 seg_size, evt_q_sz, evt_q_elts, margin = 16 << 10;
+  u32 round_rx_fifo_sz, round_tx_fifo_sz, sm_index;
   segment_manager_properties_t *props, *cprops;
-  u32 round_rx_fifo_sz, round_tx_fifo_sz;
   int rv, has_transport, seg_index;
   svm_fifo_segment_private_t *seg;
   application_t *server, *client;
@@ -1654,8 +1654,13 @@ application_local_session_connect (app_worker_t * client_wrk,
       segment_manager_segment_reader_unlock (sm);
       goto failed;
     }
+  sm_index = segment_manager_index (sm);
   ls->server_rx_fifo->ct_session_index = ls->session_index;
   ls->server_tx_fifo->ct_session_index = ls->session_index;
+  ls->server_rx_fifo->segment_manager = sm_index;
+  ls->server_tx_fifo->segment_manager = sm_index;
+  ls->server_rx_fifo->segment_index = seg_index;
+  ls->server_tx_fifo->segment_index = seg_index;
   ls->svm_segment_index = seg_index;
   ls->listener_index = ll->session_index;
   ls->client_wrk_index = client_wrk->wrk_index;
diff --git a/src/vnet/session/application_interface.c b/src/vnet/session/application_interface.c
index ace9ed2f868..d254b9bb55a 100644
--- a/src/vnet/session/application_interface.c
+++ b/src/vnet/session/application_interface.c
@@ -558,6 +558,7 @@ vnet_application_attach (vnet_app_attach_args_t * a)
 
   ASSERT (vec_len (fs->ssvm.name) <= 128);
   a->segment = &fs->ssvm;
+  a->segment_handle = segment_manager_segment_handle (sm, fs);
 
   segment_manager_segment_reader_unlock (sm);
   vec_free (app_name);
diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h
index ecd99d8755b..dfb45a78fc9 100644
--- a/src/vnet/session/application_interface.h
+++ b/src/vnet/session/application_interface.h
@@ -28,6 +28,7 @@ typedef struct _vnet_app_attach_args_t
 #undef _
   ssvm_private_t * segment;
   svm_msg_q_t *app_evt_q;
+  u64 segment_handle;
 } vnet_app_attach_args_t;
 
 typedef struct _vnet_app_detach_args_t
diff --git a/src/vnet/session/session.api b/src/vnet/session/session.api
index 3dc0cc8d360..084678f2518 100644
--- a/src/vnet/session/session.api
+++ b/src/vnet/session/session.api
@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
-option version = "1.4.0";
+option version = "1.5.0";
 
 /** \brief client->vpp, attach application to session layer
     @param client_index - opaque cookie to identify the sender
@@ -45,6 +45,7 @@ option version = "1.4.0";
     @param segment_name_length - length of segment name 
     @param segment_name - name of segment client needs to attach to
     @param app_index - index of the newly created app
+    @param segment_handle - handle for segment
 */
 define application_attach_reply {
     u32 context;
@@ -56,6 +57,7 @@ define application_attach_reply {
     u8 segment_name_length;
     u8 segment_name[128];
     u32 app_index;
+    u64 segment_handle;
 };
 
 /** \brief Application add TLS certificate
diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index d37b3c995d0..6ce9e1feac3 100755
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -825,6 +825,7 @@ done:
 	rmp->app_event_queue_address = pointer_to_uword (a->app_evt_q);
 	rmp->n_fds = n_fds;
 	rmp->fd_flags = fd_flags;
+	rmp->segment_handle = clib_host_to_net_u64 (a->segment_handle);
       }
   }));
   /* *INDENT-ON* */
diff --git a/test/test_vcl.py b/test/test_vcl.py
index 0eacf57c78e..5691604665b 100644
--- a/test/test_vcl.py
+++ b/test/test_vcl.py
@@ -347,13 +347,13 @@ class VCLThruHostStackNSessionBidirTestCase(VCLTestCase):
 
         self.thru_host_stack_setup()
         if self.vppDebug:
-            self.client_bi_dir_nsock_timeout = 120
+            self.client_bi_dir_nsock_timeout = 20
             self.client_bi_dir_nsock_test_args = ["-N", "1000", "-B", "-X",
                                                   "-I", "2",
                                                   self.loop0.local_ip4,
                                                   self.server_port]
         else:
-            self.client_bi_dir_nsock_timeout = 90
+            self.client_bi_dir_nsock_timeout = 20
             self.client_bi_dir_nsock_test_args = ["-N", "1000", "-B", "-X",
                                                   "-I", "2",
                                                   self.loop0.local_ip4,