vcl/session: send unlisten over message queue

Change-Id: I68cd6c0e6be3e8088792df3885ae190bb00462b0
Signed-off-by: Florin Coras <fcoras@cisco.com>

6 files changed, 84 insertions, 20 deletions

diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c
index b7f47d8c579..f6274e02637 100644
--- a/src/vcl/vcl_bapi.c
+++ b/src/vcl/vcl_bapi.c
@@ -340,12 +340,11 @@ static void
 vl_api_unbind_sock_reply_t_handler (vl_api_unbind_sock_reply_t * mp)
 {
   if (mp->retval)
-    clib_warning ("VCL<%d>: ERROR: sid %u: unbind failed: %U",
-		  getpid (), mp->context, format_api_error,
-		  ntohl (mp->retval));
+    VDBG (0, "ERROR: sid %u: unbind failed: %U", mp->context,
+	  format_api_error, ntohl (mp->retval));
+
+  VDBG (1, "sid %u: unbind succeeded!", mp->context);
 
-  else
-    VDBG (1, "VCL<%d>: sid %u: unbind succeeded!", getpid (), mp->context);
 }
 
 static void
@@ -605,6 +604,7 @@ vppcom_send_unbind_sock (vcl_worker_t * wrk, u64 vpp_handle)
   ump->client_index = wrk->my_client_index;
   ump->wrk_index = wrk->vpp_wrk_index;
   ump->handle = vpp_handle;
+  ump->context = wrk->wrk_index;
   vl_msg_api_send_shmem (wrk->vl_input_queue, (u8 *) & ump);
 }
 
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index f77d4479e41..28673bc7602 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -457,41 +457,35 @@ static inline void
 vcl_session_table_add_listener (vcl_worker_t * wrk, u64 listener_handle,
 				u32 value)
 {
-  /* Session and listener handles have different formats. The latter has
-   * the thread index in the upper 32 bits while the former has the session
-   * type. Knowing that, for listeners we just flip the MSB to 1 */
-  listener_handle |= 1ULL << 63;
   hash_set (wrk->session_index_by_vpp_handles, listener_handle, value);
 }
 
 static inline void
 vcl_session_table_del_listener (vcl_worker_t * wrk, u64 listener_handle)
 {
-  listener_handle |= 1ULL << 63;
   hash_unset (wrk->session_index_by_vpp_handles, listener_handle);
 }
 
 static inline vcl_session_t *
-vcl_session_table_lookup_listener (vcl_worker_t * wrk, u64 listener_handle)
+vcl_session_table_lookup_listener (vcl_worker_t * wrk, u64 handle)
 {
   uword *p;
-  u64 handle = listener_handle | (1ULL << 63);
   vcl_session_t *session;
 
   p = hash_get (wrk->session_index_by_vpp_handles, handle);
   if (!p)
     {
-      clib_warning ("VCL<%d>: couldn't find listen session: unknown vpp "
-		    "listener handle %llx", getpid (), listener_handle);
+      VDBG (0, "could not find listen session: unknown vpp listener handle"
+	    " %llx", handle);
       return 0;
     }
-  if (pool_is_free_index (wrk->sessions, p[0]))
+  session = vcl_session_get (wrk, p[0]);
+  if (!session)
     {
-      VDBG (1, "VCL<%d>: invalid listen session, sid (%u)", getpid (), p[0]);
+      VDBG (1, "invalid listen session index (%u)", p[0]);
       return 0;
     }
 
-  session = pool_elt_at_index (wrk->sessions, p[0]);
   ASSERT (session->session_state & (STATE_LISTEN | STATE_LISTEN_NO_MQ));
   return session;
 }
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index eecb2f48ce2..cc7a27642a8 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -533,6 +533,30 @@ vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp)
   return sid;
 }
 
+static void
+vcl_session_unlisten_reply_handler (vcl_worker_t * wrk, void *data)
+{
+  session_unlisten_reply_msg_t *mp = (session_unlisten_reply_msg_t *) data;
+  vcl_session_t *s;
+
+  s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
+  if (!s || s->session_state != STATE_DISCONNECT)
+    {
+      VDBG (0, "Unlisten reply with wrong handle %llx", mp->handle);
+      return;
+    }
+
+  if (mp->retval)
+    VDBG (0, "ERROR: session %u [0xllx]: unlisten failed: %U",
+	  s->session_index, mp->handle, format_api_error, ntohl (mp->retval));
+
+  if (mp->context != wrk->wrk_index)
+    VDBG (0, "wrong context");
+
+  vcl_session_table_del_vpp_handle (wrk, mp->handle);
+  vcl_session_free (wrk, s);
+}
+
 static vcl_session_t *
 vcl_session_accepted (vcl_worker_t * wrk, session_accepted_msg_t * msg)
 {
@@ -670,6 +694,9 @@ vcl_handle_mq_event (vcl_worker_t * wrk, session_event_t * e)
     case SESSION_CTRL_EVT_BOUND:
       vcl_session_bound_handler (wrk, (session_bound_msg_t *) e->data);
       break;
+    case SESSION_CTRL_EVT_UNLISTEN_REPLY:
+      vcl_session_unlisten_reply_handler (wrk, e->data);
+      break;
     case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
       vcl_session_req_worker_update_handler (wrk, e->data);
       break;
@@ -821,7 +848,6 @@ vppcom_session_unbind (u32 session_handle)
     return VPPCOM_EBADFD;
 
   vpp_handle = session->vpp_handle;
-  vcl_session_table_del_listener (wrk, vpp_handle);
   session->vpp_handle = ~0;
   session->session_state = STATE_DISCONNECT;
 
@@ -1079,6 +1105,7 @@ vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * session,
 	    VDBG (0, "session %u [0x%llx]: listener unbind failed! "
 		  "rv %d (%s)", session->session_index, vpp_handle, rv,
 		  vppcom_retval_str (rv));
+	  return rv;
 	}
       else if (state & STATE_OPEN)
 	{
@@ -1937,6 +1964,9 @@ vcl_select_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
 	  *bits_set += 1;
 	}
       break;
+    case SESSION_CTRL_EVT_UNLISTEN_REPLY:
+      vcl_session_unlisten_reply_handler (wrk, e->data);
+      break;
     case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
       vcl_session_worker_update_reply_handler (wrk, e->data);
       break;
@@ -2558,6 +2588,9 @@ vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
       events[*num_ev].events |= EPOLLHUP | EPOLLRDHUP;
       session_evt_data = session->vep.ev.data.u64;
       break;
+    case SESSION_CTRL_EVT_UNLISTEN_REPLY:
+      vcl_session_unlisten_reply_handler (wrk, e->data);
+      break;
     case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
       vcl_session_req_worker_update_handler (wrk, e->data);
       break;
diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h
index 18878cdf568..441b5f613a7 100644
--- a/src/vnet/session/application_interface.h
+++ b/src/vnet/session/application_interface.h
@@ -95,7 +95,7 @@ typedef struct _vnet_bind_args_t
   u64 handle;
 } vnet_listen_args_t;
 
-typedef struct _vnet_unbind_args_t
+typedef struct _vnet_unlisten_args_t
 {
   union
   {
@@ -268,6 +268,13 @@ typedef struct session_bound_msg_
   u8 segment_name[128];
 } __clib_packed session_bound_msg_t;
 
+typedef struct session_unlisten_reply_msg_
+{
+  u32 context;
+  u64 handle;
+  i32 retval;
+} __clib_packed session_unlisten_reply_msg_t;
+
 typedef struct session_accepted_msg_
 {
   u32 context;
diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index 0285691182e..f28c12c7c29 100755
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -1248,7 +1248,8 @@ vl_api_unbind_sock_t_handler (vl_api_unbind_sock_t * mp)
 {
   vl_api_unbind_sock_reply_t *rmp;
   vnet_unlisten_args_t _a, *a = &_a;
-  application_t *app;
+  app_worker_t *app_wrk;
+  application_t *app = 0;
   int rv = 0;
 
   if (session_manager_is_enabled () == 0)
@@ -1269,6 +1270,34 @@ vl_api_unbind_sock_t_handler (vl_api_unbind_sock_t * mp)
 
 done:
   REPLY_MACRO (VL_API_UNBIND_SOCK_REPLY);
+
+  /*
+   * Send reply over msg queue
+   */
+  svm_msg_q_msg_t _msg, *msg = &_msg;
+  session_unlisten_reply_msg_t *ump;
+  svm_msg_q_t *app_mq;
+  session_event_t *evt;
+
+  if (!app)
+    return;
+
+  app_wrk = application_get_worker (app, a->wrk_map_index);
+  if (!app_wrk)
+    return;
+
+  app_mq = app_wrk->event_queue;
+  if (mq_try_lock_and_alloc_msg (app_mq, msg))
+    return;
+
+  evt = svm_msg_q_msg_data (app_mq, msg);
+  clib_memset (evt, 0, sizeof (*evt));
+  evt->event_type = SESSION_CTRL_EVT_UNLISTEN_REPLY;
+  ump = (session_unlisten_reply_msg_t *) evt->data;
+  ump->context = mp->context;
+  ump->handle = mp->handle;
+  ump->retval = rv;
+  svm_msg_q_add_and_unlock (app_mq, msg);
 }
 
 static void
diff --git a/src/vnet/session/session_types.h b/src/vnet/session/session_types.h
index efa3dea950c..97c915bbae2 100644
--- a/src/vnet/session/session_types.h
+++ b/src/vnet/session/session_types.h
@@ -392,6 +392,7 @@ typedef enum
   FIFO_EVENT_BUILTIN_TX,
   FIFO_EVENT_RPC,
   SESSION_CTRL_EVT_BOUND,
+  SESSION_CTRL_EVT_UNLISTEN_REPLY,
   SESSION_CTRL_EVT_ACCEPTED,
   SESSION_CTRL_EVT_ACCEPTED_REPLY,
   SESSION_CTRL_EVT_CONNECTED,