session: improve close procedure

Change-Id: I6201a044e70ab6a58db8212960c57edc77c41f96
Signed-off-by: Florin Coras <fcoras@cisco.com>

6 files changed, 56 insertions, 19 deletions

diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index b47a633c2cc..17944536b6c 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -445,6 +445,14 @@ vlib_get_worker_vlib_main (u32 worker_index)
   return vm;
 }
 
+static inline u8
+vlib_thread_is_main_w_barrier (void)
+{
+  return (!vlib_num_workers ()
+	  || ((vlib_get_thread_index () == 0
+	       && vlib_worker_threads->wait_at_barrier[0])));
+}
+
 static inline void
 vlib_put_frame_queue_elt (vlib_frame_queue_elt_t * hf)
 {
diff --git a/src/vnet/session/application_interface.c b/src/vnet/session/application_interface.c
index 72a9985dced..d35a8293723 100644
--- a/src/vnet/session/application_interface.c
+++ b/src/vnet/session/application_interface.c
@@ -82,9 +82,7 @@ const char test_srv_key_rsa[] =
 const u32 test_srv_key_rsa_len = sizeof (test_srv_key_rsa);
 
 #define app_interface_check_thread_and_barrier(_fn, _arg)		\
-  if (PREDICT_FALSE (vlib_get_thread_index () != 0 ||			\
-                     (vlib_worker_threads[0].wait_at_barrier &&		\
-		      !vlib_worker_threads[0].wait_at_barrier[0])))	\
+  if (PREDICT_FALSE (!vlib_thread_is_main_w_barrier ()))		\
     {									\
       vlib_rpc_call_main_thread (_fn, (u8 *) _arg, sizeof(*_arg));	\
       return 0;								\
diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c
index 0c19293ce3b..bf62040271f 100644
--- a/src/vnet/session/segment_manager.c
+++ b/src/vnet/session/segment_manager.c
@@ -591,7 +591,9 @@ segment_manager_dealloc_fifos (u32 segment_index, svm_fifo_t * rx_fifo,
       /* Remove segment manager if no sessions and detached from app */
       if (segment_manager_app_detached (sm)
 	  && !segment_manager_has_fifos (sm))
-	segment_manager_del (sm);
+	{
+	  segment_manager_del (sm);
+	}
     }
   else
     segment_manager_segment_reader_unlock (sm);
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index 133c91ca198..3eaf9846709 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -152,6 +152,14 @@ session_free (stream_session_t * s)
     memset (s, 0xFA, sizeof (*s));
 }
 
+static void
+session_free_w_fifos (stream_session_t * s)
+{
+  segment_manager_dealloc_fifos (s->svm_segment_index, s->server_rx_fifo,
+				 s->server_tx_fifo);
+  session_free (s);
+}
+
 int
 session_alloc_fifos (segment_manager_t * sm, stream_session_t * s)
 {
@@ -751,7 +759,7 @@ stream_session_disconnect_notify (transport_connection_t * tc)
   stream_session_t *s;
 
   s = session_get (tc->s_index, tc->thread_index);
-  s->session_state = SESSION_STATE_CLOSING;
+  s->session_state = SESSION_STATE_TRANSPORT_CLOSING;
   app_wrk = app_worker_get_if_valid (s->app_wrk_index);
   if (!app_wrk)
     return;
@@ -773,10 +781,7 @@ stream_session_delete (stream_session_t * s)
   if ((rv = session_lookup_del_session (s)))
     clib_warning ("hash delete error, rv %d", rv);
 
-  /* Cleanup fifo segments */
-  segment_manager_dealloc_fifos (s->svm_segment_index, s->server_rx_fifo,
-				 s->server_tx_fifo);
-  session_free (s);
+  session_free_w_fifos (s);
 }
 
 /**
@@ -793,10 +798,30 @@ stream_session_delete_notify (transport_connection_t * tc)
   stream_session_t *s;
 
   /* App might've been removed already */
-  s = session_get_if_valid (tc->s_index, tc->thread_index);
-  if (!s)
+  if (!(s = session_get_if_valid (tc->s_index, tc->thread_index)))
     return;
-  stream_session_delete (s);
+
+  /* Make sure we don't try to send anything more */
+  svm_fifo_dequeue_drop_all (s->server_tx_fifo);
+
+  switch (s->session_state)
+    {
+    case SESSION_STATE_TRANSPORT_CLOSING:
+      /* If transport finishes or times out before we get a reply
+       * from the app, do the whole disconnect since we might still
+       * have lingering events */
+      stream_session_disconnect (s);
+      break;
+    case SESSION_STATE_CLOSING:
+      /* Cleanup lookup table. Transport needs to still be valid */
+      session_lookup_del_session (s);
+      break;
+    case SESSION_STATE_CLOSED:
+      stream_session_delete (s);
+      break;
+    }
+
+  s->session_state = SESSION_STATE_CLOSED;
 }
 
 /**
@@ -1050,11 +1075,9 @@ stream_session_disconnect (stream_session_t * s)
   s->session_state = SESSION_STATE_CLOSING;
 
   /* If we are in the handler thread, or being called with the worker barrier
-   * held (api/cli), just append a new event to pending disconnects vector. */
-  if ((thread_index == 0 && !vlib_get_current_process (vlib_get_main ()))
-      || thread_index == s->thread_index)
+   * held, just append a new event to pending disconnects vector. */
+  if (vlib_thread_is_main_w_barrier () || thread_index == s->thread_index)
     {
-      ASSERT (s->thread_index == thread_index || thread_index == 0);
       vec_add2 (smm->pending_disconnects[s->thread_index], evt, 1);
       memset (evt, 0, sizeof (*evt));
       evt->session_handle = session_handle (s);
@@ -1074,6 +1097,12 @@ stream_session_disconnect (stream_session_t * s)
 void
 stream_session_disconnect_transport (stream_session_t * s)
 {
+  /* If transport is already closed, just free the session */
+  if (s->session_state == SESSION_STATE_CLOSED)
+    {
+      session_free_w_fifos (s);
+      return;
+    }
   s->session_state = SESSION_STATE_CLOSED;
   tp_vfts[session_get_transport_proto (s)].close (s->connection_index,
 						  s->thread_index);
@@ -1097,9 +1126,7 @@ stream_session_cleanup (stream_session_t * s)
 						    s->thread_index);
   /* Since we called cleanup, no delete notification will come. So, make
    * sure the session is properly freed. */
-  segment_manager_dealloc_fifos (s->svm_segment_index, s->server_rx_fifo,
-				 s->server_tx_fifo);
-  session_free (s);
+  session_free_w_fifos (s);
 }
 
 transport_service_type_t
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h
index 19609da89e5..aa827556d45 100644
--- a/src/vnet/session/session.h
+++ b/src/vnet/session/session.h
@@ -561,6 +561,7 @@ int session_send_io_evt_to_thread_custom (svm_fifo_t * f, u32 thread_index,
 					  session_evt_type_t evt_type);
 void session_send_rpc_evt_to_thread (u32 thread_index, void *fp,
 				     void *rpc_args);
+
 ssvm_private_t *session_manager_get_evt_q_segment (void);
 
 u8 *format_stream_session (u8 * s, va_list * args);
diff --git a/src/vnet/session/stream_session.h b/src/vnet/session/stream_session.h
index b08f9592510..30178d7a453 100644
--- a/src/vnet/session/stream_session.h
+++ b/src/vnet/session/stream_session.h
@@ -31,6 +31,7 @@ typedef enum
   SESSION_STATE_ACCEPTING,
   SESSION_STATE_READY,
   SESSION_STATE_OPENED,
+  SESSION_STATE_TRANSPORT_CLOSING,
   SESSION_STATE_CLOSING,
   SESSION_STATE_CLOSED,
   SESSION_STATE_N_STATES,