vcl/session: use mq for bind replies

Change-Id: Iac6e1c32cf99c5392a29f7366401b7fc39e463e3
Signed-off-by: Florin Coras <fcoras@cisco.com>

7 files changed, 184 insertions, 57 deletions

diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c
index 2b369fe0c72..d9e8e7e046c 100644
--- a/src/vcl/vcl_bapi.c
+++ b/src/vcl/vcl_bapi.c
@@ -221,57 +221,9 @@ static void
 static void
 vl_api_bind_sock_reply_t_handler (vl_api_bind_sock_reply_t * mp)
 {
-  vcl_session_t *session = 0;
-  u32 session_index = mp->context;
-  int rv;
-
-  VCL_SESSION_LOCK_AND_GET (session_index, &session);
-done:
-  if (mp->retval)
-    {
-      clib_warning ("VCL<%d>: ERROR: vpp handle 0x%llx, "
-		    "sid %u: bind failed: %U",
-		    getpid (), mp->handle, session_index,
-		    format_api_error, ntohl (mp->retval));
-      rv = vppcom_session_at_index (session_index, &session);
-      if (rv == VPPCOM_OK)
-	{
-	  session->session_state = STATE_FAILED;
-	  session->vpp_handle = mp->handle;
-	}
-      else
-	{
-	  clib_warning ("[%s] ERROR: vpp handle 0x%llx, sid %u: "
-			"Invalid session index (%u)!",
-			getpid (), mp->handle, session_index);
-	}
-      goto done_unlock;
-    }
-
-  session->vpp_handle = mp->handle;
-  session->transport.is_ip4 = mp->lcl_is_ip4;
-  clib_memcpy (&session->transport.lcl_ip, mp->lcl_ip,
-	       sizeof (ip46_address_t));
-  session->transport.lcl_port = mp->lcl_port;
-  vppcom_session_table_add_listener (mp->handle, session_index);
-  session->session_state = STATE_LISTEN;
-
-  if (session->is_dgram)
-    {
-      svm_fifo_t *rx_fifo, *tx_fifo;
-      session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *);
-      rx_fifo = uword_to_pointer (mp->rx_fifo, svm_fifo_t *);
-      rx_fifo->client_session_index = session_index;
-      tx_fifo = uword_to_pointer (mp->tx_fifo, svm_fifo_t *);
-      tx_fifo->client_session_index = session_index;
-      session->rx_fifo = rx_fifo;
-      session->tx_fifo = tx_fifo;
-    }
-
-  VDBG (1, "VCL<%d>: vpp handle 0x%llx, sid %u: bind succeeded!",
-	getpid (), mp->handle, mp->context);
-done_unlock:
-  VCL_SESSION_UNLOCK ();
+  /* Expecting a similar message on mq. So ignore this */
+  VDBG (1, "VCL<%d>: bapi msg vpp handle 0x%llx, sid %u: bind retval: %u!",
+	getpid (), mp->handle, mp->context, mp->retval);
 }
 
 static void
@@ -299,8 +251,8 @@ _(APP_CUT_THROUGH_REGISTRATION_ADD, app_cut_through_registration_add)	\
 void
 vppcom_api_hookup (void)
 {
-#define _(N, n)                                                  \
-    vl_msg_api_set_handlers(VL_API_##N, #n,                     \
+#define _(N, n)                                                	\
+    vl_msg_api_set_handlers(VL_API_##N, #n,                    	\
                            vl_api_##n##_t_handler,              \
                            vl_noop_handler,                     \
                            vl_api_##n##_t_endian,               \
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index f95e72fc94e..05c1dbbb501 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -466,7 +466,7 @@ done_unlock:
 }
 
 static u32
-vcl_reset_handler (session_reset_msg_t * reset_msg)
+vcl_session_reset_handler (session_reset_msg_t * reset_msg)
 {
   vcl_session_t *session;
   u32 sid;
@@ -485,6 +485,57 @@ vcl_reset_handler (session_reset_msg_t * reset_msg)
   return sid;
 }
 
+static u32
+vcl_session_bound_handler (session_bound_msg_t * mp)
+{
+  vcl_session_t *session;
+  u32 sid = mp->context;
+
+  session = vcl_session_get (sid);
+  if (mp->retval)
+    {
+      VDBG (0, "VCL<%d>: ERROR: vpp handle 0x%llx, sid %u: bind failed: %U",
+	    getpid (), mp->handle, sid, format_api_error, ntohl (mp->retval));
+      if (session)
+	{
+	  session->session_state = STATE_FAILED;
+	  session->vpp_handle = mp->handle;
+	  return sid;
+	}
+      else
+	{
+	  clib_warning ("[%s] ERROR: vpp handle 0x%llx, sid %u: "
+			"Invalid session index (%u)!",
+			getpid (), mp->handle, sid);
+	  return VCL_INVALID_SESSION_INDEX;
+	}
+    }
+
+  session->vpp_handle = mp->handle;
+  session->transport.is_ip4 = mp->lcl_is_ip4;
+  clib_memcpy (&session->transport.lcl_ip, mp->lcl_ip,
+	       sizeof (ip46_address_t));
+  session->transport.lcl_port = mp->lcl_port;
+  vppcom_session_table_add_listener (mp->handle, sid);
+  session->session_state = STATE_LISTEN;
+
+  if (session->is_dgram)
+    {
+      svm_fifo_t *rx_fifo, *tx_fifo;
+      session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *);
+      rx_fifo = uword_to_pointer (mp->rx_fifo, svm_fifo_t *);
+      rx_fifo->client_session_index = sid;
+      tx_fifo = uword_to_pointer (mp->tx_fifo, svm_fifo_t *);
+      tx_fifo->client_session_index = sid;
+      session->rx_fifo = rx_fifo;
+      session->tx_fifo = tx_fifo;
+    }
+
+  VDBG (1, "VCL<%d>: vpp handle 0x%llx, sid %u: bind succeeded!",
+	getpid (), mp->handle, sid);
+  return sid;
+}
+
 int
 vcl_handle_mq_ctrl_event (session_event_t * e)
 {
@@ -534,7 +585,10 @@ vcl_handle_mq_ctrl_event (session_event_t * e)
 	    sid);
       break;
     case SESSION_CTRL_EVT_RESET:
-      vcl_reset_handler ((session_reset_msg_t *) e->data);
+      vcl_session_reset_handler ((session_reset_msg_t *) e->data);
+      break;
+    case SESSION_CTRL_EVT_BOUND:
+      vcl_session_bound_handler ((session_bound_msg_t *) e->data);
       break;
     default:
       clib_warning ("unhandled %u", e->event_type);
@@ -1780,7 +1834,7 @@ vcl_select_handle_mq (svm_msg_q_t * mq, unsigned long n_bits,
 	    }
 	  break;
 	case SESSION_CTRL_EVT_RESET:
-	  sid = vcl_reset_handler ((session_reset_msg_t *) e->data);
+	  sid = vcl_session_reset_handler ((session_reset_msg_t *) e->data);
 	  if (sid < n_bits && except_map)
 	    {
 	      clib_bitmap_set_no_check (except_map, sid, 1);
@@ -2404,7 +2458,7 @@ vcl_epoll_wait_handle_mq (svm_msg_q_t * mq, struct epoll_event *events,
 	  clib_spinlock_unlock (&vcm->sessions_lockp);
 	  break;
 	case SESSION_CTRL_EVT_RESET:
-	  sid = vcl_reset_handler ((session_reset_msg_t *) e->data);
+	  sid = vcl_session_reset_handler ((session_reset_msg_t *) e->data);
 	  if (!(session = vcl_session_get (sid)))
 	    break;
 	  add_event = 1;
diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c
index 1ad94cc2778..b7498d00881 100644
--- a/src/vnet/session/application.c
+++ b/src/vnet/session/application.c
@@ -769,6 +769,12 @@ application_has_global_scope (application_t * app)
   return app->flags & APP_OPTIONS_FLAGS_USE_GLOBAL_SCOPE;
 }
 
+u8
+application_use_mq_for_ctrl (application_t * app)
+{
+  return app->flags & APP_OPTIONS_FLAGS_USE_MQ_FOR_CTRL_MSGS;
+}
+
 /**
  * Send an API message to the external app, to map new segment
  */
@@ -1163,6 +1169,17 @@ application_get_local_session_from_handle (session_handle_t handle)
   return application_get_local_session (server_wrk, session_index);
 }
 
+local_session_t *
+application_get_local_listen_session_from_handle (session_handle_t lh)
+{
+  u32 ll_index, server_wrk_index;
+  app_worker_t *server_wrk;
+
+  local_session_parse_handle (lh, &server_wrk_index, &ll_index);
+  server_wrk = app_worker_get (server_wrk_index);
+  return application_get_local_listen_session (server_wrk, ll_index);
+}
+
 always_inline void
 application_local_listener_session_endpoint (local_session_t * ll,
 					     session_endpoint_t * sep)
diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h
index cf855660ca5..de609d2d7a0 100644
--- a/src/vnet/session/application.h
+++ b/src/vnet/session/application.h
@@ -247,6 +247,7 @@ u32 application_local_session_table (application_t * app);
 u8 *application_name_from_index (u32 app_wrk_index);
 u8 application_has_local_scope (application_t * app);
 u8 application_has_global_scope (application_t * app);
+u8 application_use_mq_for_ctrl (application_t * app);
 void application_setup_proxy (application_t * app);
 void application_remove_proxy (application_t * app);
 
@@ -262,6 +263,8 @@ local_session_t *application_get_local_session (app_worker_t * app,
 						u32 session_index);
 local_session_t *application_get_local_session_from_handle (session_handle_t
 							    handle);
+local_session_t
+  * application_get_local_listen_session_from_handle (session_handle_t lh);
 int application_start_local_listen (app_worker_t * server,
 				    session_endpoint_t * sep,
 				    session_handle_t * handle);
diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h
index 13ec73adf0f..1f481dcf85e 100644
--- a/src/vnet/session/application_interface.h
+++ b/src/vnet/session/application_interface.h
@@ -209,6 +209,22 @@ typedef struct
 #undef _
 } app_session_t;
 
+typedef struct session_bound_msg_
+{
+  u32 context;
+  u64 handle;
+  i32 retval;
+  u8 lcl_is_ip4;
+  u8 lcl_ip[16];
+  u16 lcl_port;
+  u64 rx_fifo;
+  u64 tx_fifo;
+  u64 vpp_evt_q;
+  u32 segment_size;
+  u8 segment_name_length;
+  u8 segment_name[128];
+} __clib_packed session_bound_msg_t;
+
 typedef struct session_accepted_msg_
 {
   u32 context;
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h
index 63ed83d40df..184fa997736 100644
--- a/src/vnet/session/session.h
+++ b/src/vnet/session/session.h
@@ -39,6 +39,7 @@ typedef enum
   FIFO_EVENT_DISCONNECT,
   FIFO_EVENT_BUILTIN_RX,
   FIFO_EVENT_RPC,
+  SESSION_CTRL_EVT_BOUND,
   SESSION_CTRL_EVT_ACCEPTED,
   SESSION_CTRL_EVT_ACCEPTED_REPLY,
   SESSION_CTRL_EVT_CONNECTED,
diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index c12354fdeae..13a337699d2 100755
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -636,6 +636,72 @@ done:
   return 0;
 }
 
+static int
+mq_send_session_bound_cb (u32 app_wrk_index, u32 api_context,
+			  session_handle_t handle, int rv)
+{
+  svm_msg_q_msg_t _msg, *msg = &_msg;
+  svm_msg_q_t *app_mq, *vpp_evt_q;
+  transport_connection_t *tc;
+  stream_session_t *ls = 0;
+  session_bound_msg_t *mp;
+  app_worker_t *app_wrk;
+  session_event_t *evt;
+  application_t *app;
+
+  app_wrk = app_worker_get (app_wrk_index);
+  app = application_get (app_wrk->app_index);
+  app_mq = app_wrk->event_queue;
+  if (!app_mq)
+    {
+      clib_warning ("app %u with api index: %u not attached", app->app_index,
+		    app->api_client_index);
+      return -1;
+    }
+
+  svm_msg_q_lock_and_alloc_msg_w_ring (app_mq, SESSION_MQ_CTRL_EVT_RING,
+				       SVM_Q_WAIT, msg);
+  svm_msg_q_unlock (app_mq);
+  evt = svm_msg_q_msg_data (app_mq, msg);
+  memset (evt, 0, sizeof (*evt));
+  evt->event_type = SESSION_CTRL_EVT_BOUND;
+  mp = (session_bound_msg_t *) evt->data;
+  mp->context = api_context;
+
+  if (rv)
+    goto done;
+
+  mp->handle = handle;
+  if (application_has_global_scope (app))
+    {
+      ls = listen_session_get_from_handle (handle);
+      tc = listen_session_get_transport (ls);
+      mp->lcl_port = tc->lcl_port;
+      mp->lcl_is_ip4 = tc->is_ip4;
+      clib_memcpy (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip));
+    }
+  else
+    {
+      local_session_t *local;
+      local = application_get_local_listen_session_from_handle (handle);
+      mp->lcl_port = local->port;
+      mp->lcl_is_ip4 = session_type_is_ip4 (local->session_type);
+    }
+
+  if (ls && session_transport_service_type (ls) == TRANSPORT_SERVICE_CL)
+    {
+      mp->rx_fifo = pointer_to_uword (ls->server_rx_fifo);
+      mp->tx_fifo = pointer_to_uword (ls->server_tx_fifo);
+      vpp_evt_q = session_manager_get_vpp_event_queue (0);
+      mp->vpp_evt_q = pointer_to_uword (vpp_evt_q);
+    }
+
+done:
+  mp->retval = rv;
+  svm_msg_q_add (app_mq, msg, SVM_Q_WAIT);
+  return 0;
+}
+
 static session_cb_vft_t session_mq_cb_vft = {
   .session_accept_callback = mq_send_session_accepted_cb,
   .session_disconnect_callback = mq_send_session_disconnected_cb,
@@ -791,6 +857,7 @@ vl_api_bind_uri_t_handler (vl_api_bind_uri_t * mp)
   stream_session_t *s;
   application_t *app = 0;
   svm_msg_q_t *vpp_evt_q;
+  app_worker_t *app_wrk;
   int rv;
 
   if (session_manager_is_enabled () == 0)
@@ -837,6 +904,14 @@ done:
       }
   }));
   /* *INDENT-ON* */
+
+  /* If app uses mq for control messages, send an mq message as well */
+  if (app && application_use_mq_for_ctrl (app))
+    {
+      app_wrk = application_get_worker (app, 0);
+      mq_send_session_bound_cb (app_wrk->wrk_index, mp->context, a->handle,
+				rv);
+    }
 }
 
 static void
@@ -1070,6 +1145,7 @@ vl_api_bind_sock_t_handler (vl_api_bind_sock_t * mp)
   int rv = 0;
   clib_error_t *error;
   application_t *app = 0;
+  app_worker_t *app_wrk;
   stream_session_t *s;
   transport_connection_t *tc = 0;
   ip46_address_t *ip46;
@@ -1129,6 +1205,14 @@ done:
       }
   }));
   /* *INDENT-ON* */
+
+  /* If app uses mq for control messages, send an mq message as well */
+  if (app && application_use_mq_for_ctrl (app))
+    {
+      app_wrk = application_get_worker (app, mp->wrk_index);
+      mq_send_session_bound_cb (app_wrk->wrk_index, mp->context, a->handle,
+				rv);
+    }
 }
 
 static void