session: per app wrk client ct segment handle

Make sure ct client segment handles do not collide if multi worker
application establishes cut-through sessions to only one server segment
manager.

Type: fix

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I905379f9ed73c64d57a826a3e97d53dab3a87517

3 files changed, 22 insertions, 19 deletions

diff --git a/src/vnet/session/application_local.c b/src/vnet/session/application_local.c
index e3451531a48..b14386a9b00 100644
--- a/src/vnet/session/application_local.c
+++ b/src/vnet/session/application_local.c
@@ -194,6 +194,12 @@ ct_set_invalid_app_wrk (ct_connection_t *ct, u8 is_client)
     }
 }
 
+static inline u64
+ct_client_seg_handle (u64 server_sh, u32 client_wrk_index)
+{
+  return (((u64) client_wrk_index << 56) | server_sh);
+}
+
 static void
 ct_session_dealloc_fifos (ct_connection_t *ct, svm_fifo_t *rx_fifo,
 			  svm_fifo_t *tx_fifo)
@@ -314,7 +320,8 @@ ct_session_dealloc_fifos (ct_connection_t *ct, svm_fifo_t *rx_fifo,
       segment_manager_t *csm;
       csm = app_worker_get_connect_segment_manager (app_wrk);
       if (!segment_manager_app_detached (csm))
-	app_worker_del_segment_notify (app_wrk, ct->segment_handle);
+	app_worker_del_segment_notify (
+	  app_wrk, ct_client_seg_handle (ct->segment_handle, ct->client_wrk));
     }
 
   /* Notify server app and free segment */
@@ -455,11 +462,11 @@ ct_alloc_segment (ct_main_t *cm, app_worker_t *server_wrk, u64 table_handle,
 		  segment_manager_t *sm, u32 client_wrk_index)
 {
   u32 seg_ctx_index = ~0, sm_index, pair_bytes;
+  u64 seg_size, seg_handle, client_seg_handle;
   segment_manager_props_t *props;
   const u32 margin = 16 << 10;
   ct_segments_ctx_t *seg_ctx;
   app_worker_t *client_wrk;
-  u64 seg_size, seg_handle;
   application_t *server;
   ct_segment_t *ct_seg;
   uword *spp;
@@ -521,7 +528,11 @@ ct_alloc_segment (ct_main_t *cm, app_worker_t *server_wrk, u64 table_handle,
     goto error;
 
   client_wrk = app_worker_get (client_wrk_index);
-  if (app_worker_add_segment_notify (client_wrk, seg_handle))
+  /* Make sure client workers do not have overlapping segment handles.
+   * Ideally, we should attach fs to client worker segment manager and
+   * create a new handle but that's not currently possible. */
+  client_seg_handle = ct_client_seg_handle (seg_handle, client_wrk_index);
+  if (app_worker_add_segment_notify (client_wrk, client_seg_handle))
     {
       app_worker_del_segment_notify (server_wrk, seg_handle);
       goto error;
@@ -738,7 +749,8 @@ ct_accept_one (u32 thread_index, u32 ho_index)
   cct->client_tx_fifo = ss->rx_fifo;
   cct->client_rx_fifo->refcnt++;
   cct->client_tx_fifo->refcnt++;
-  cct->segment_handle = sct->segment_handle;
+  cct->segment_handle =
+    ct_client_seg_handle (sct->segment_handle, cct->client_wrk);
 
   session_set_state (ss, SESSION_STATE_ACCEPTING);
   if (app_worker_accept_notify (server_wrk, ss))
diff --git a/src/vnet/session/segment_manager.h b/src/vnet/session/segment_manager.h
index e786b3144c2..1e99c4605a6 100644
--- a/src/vnet/session/segment_manager.h
+++ b/src/vnet/session/segment_manager.h
@@ -190,7 +190,9 @@ static inline void
 segment_manager_parse_segment_handle (u64 segment_handle, u32 * sm_index,
 				      u32 * segment_index)
 {
-  *sm_index = segment_handle >> 32;
+  /* Upper 8 bits zeroed out as they may be used for cut-through segments.
+   * See @ref ct_alloc_segment */
+  *sm_index = (segment_handle >> 32) & 0xFFFFFF;
   *segment_index = segment_handle & 0xFFFFFFFF;
 }
 
diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index 55fc72ee4c2..ff49d2ac032 100644
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -212,7 +212,6 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context,
 			      session_t * s, session_error_t err)
 {
   session_connected_msg_t m = { 0 };
-  transport_connection_t *tc;
   fifo_segment_t *eq_seg;
   app_worker_t *app_wrk;
   application_t *app;
@@ -230,14 +229,6 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context,
 
   if (session_has_transport (s))
     {
-      tc = session_get_transport (s);
-      if (!tc)
-	{
-	  clib_warning ("failed to retrieve transport!");
-	  m.retval = SESSION_E_REFUSED;
-	  goto snd_msg;
-	}
-
       m.handle = session_handle (s);
       m.vpp_event_queue_address =
 	fifo_segment_msg_q_offset (eq_seg, s->thread_index);
@@ -252,7 +243,6 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context,
   else
     {
       ct_connection_t *cct;
-      session_t *ss;
 
       cct = (ct_connection_t *) session_get_transport (s);
       m.handle = session_handle (s);
@@ -263,11 +253,10 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context,
       m.server_rx_fifo = fifo_segment_fifo_offset (s->rx_fifo);
       m.server_tx_fifo = fifo_segment_fifo_offset (s->tx_fifo);
       m.segment_handle = session_segment_handle (s);
-      ss = ct_session_get_peer (s);
-      m.ct_rx_fifo = fifo_segment_fifo_offset (ss->tx_fifo);
-      m.ct_tx_fifo = fifo_segment_fifo_offset (ss->rx_fifo);
-      m.ct_segment_handle = session_segment_handle (ss);
       m.mq_index = s->thread_index;
+      m.ct_rx_fifo = fifo_segment_fifo_offset (cct->client_rx_fifo);
+      m.ct_tx_fifo = fifo_segment_fifo_offset (cct->client_tx_fifo);
+      m.ct_segment_handle = cct->segment_handle;
     }
 
   /* Setup client session index in advance, in case data arrives