tcp/session: sprinkle prefetches

Change-Id: Idef3c665580c13d72e99f43d16b8b13cc6ab746f
Signed-off-by: Florin Coras <fcoras@cisco.com>

4 files changed, 37 insertions, 16 deletions

diff --git a/src/vnet/session-apps/echo_server.c b/src/vnet/session-apps/echo_server.c
index 9f34d4c0a5e..0bde2e4995f 100644
--- a/src/vnet/session-apps/echo_server.c
+++ b/src/vnet/session-apps/echo_server.c
@@ -375,6 +375,7 @@ echo_server_create (vlib_main_t * vm, u8 * appns_id, u64 appns_flags,
   vec_validate (esm->rx_buf, num_threads - 1);
   vec_validate (esm->rx_retries, num_threads - 1);
 
+  esm->rcv_buffer_size = clib_max (esm->rcv_buffer_size, esm->fifo_size);
   for (i = 0; i < num_threads; i++)
     vec_validate (esm->rx_buf[i], esm->rcv_buffer_size);
 
diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index 9cd0ef18415..b8f429eb1d7 100644
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -144,7 +144,7 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
   u8 *data0;
   int i, n_bytes_read;
   u32 n_bytes_per_buf, deq_per_buf, deq_per_first_buf;
-  u32 buffers_allocated, buffers_allocated_this_call;
+  u32 bufs_alloc, bufs_now;
 
   next_index = next0 = smm->session_type_to_next[s0->session_type];
 
@@ -218,23 +218,19 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
 	{
 	  vec_validate (smm->tx_buffers[thread_index],
 			n_bufs + n_bufs_per_frame - 1);
-	  buffers_allocated = 0;
+	  bufs_alloc = 0;
 	  do
 	    {
-	      buffers_allocated_this_call = vlib_buffer_alloc (vm,
-							       &smm->tx_buffers
-							       [thread_index]
-							       [n_bufs +
-								buffers_allocated],
-							       n_bufs_per_frame
-							       -
-							       buffers_allocated);
-	      buffers_allocated += buffers_allocated_this_call;
+	      bufs_now =
+		vlib_buffer_alloc (vm,
+				   &smm->tx_buffers[thread_index][n_bufs +
+								  bufs_alloc],
+				   n_bufs_per_frame - bufs_alloc);
+	      bufs_alloc += bufs_now;
 	    }
-	  while (buffers_allocated_this_call > 0
-		 && ((buffers_allocated + n_bufs < n_bufs_per_frame)));
+	  while (bufs_now > 0 && ((bufs_alloc + n_bufs < n_bufs_per_frame)));
 
-	  n_bufs += buffers_allocated;
+	  n_bufs += bufs_alloc;
 	  _vec_len (smm->tx_buffers[thread_index]) = n_bufs;
 
 	  if (PREDICT_FALSE (n_bufs < n_bufs_per_frame))
@@ -252,6 +248,15 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
 	   * Handle first buffer in chain separately
 	   */
 
+	  len_to_deq0 = clib_min (left_to_snd0, deq_per_first_buf);
+	  if (left_to_snd0 > len_to_deq0 && n_left_to_next > 1)
+	    {
+	      vlib_buffer_t *pb;
+	      u32 pbi = smm->tx_buffers[thread_index][n_bufs - 2];
+	      pb = vlib_get_buffer (vm, pbi);
+	      vlib_prefetch_buffer_header (pb, LOAD);
+	    }
+
 	  /* Get free buffer */
 	  ASSERT (n_bufs >= 1);
 	  bi0 = smm->tx_buffers[thread_index][--n_bufs];
@@ -264,11 +269,10 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 	  b0 = vlib_get_buffer (vm, bi0);
 	  b0->error = 0;
-	  b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
+	  b0->flags = VNET_BUFFER_F_LOCALLY_ORIGINATED;
 	  b0->current_data = 0;
 	  b0->total_length_not_including_first_buffer = 0;
 
-	  len_to_deq0 = clib_min (left_to_snd0, deq_per_first_buf);
 	  data0 = vlib_buffer_make_headroom (b0, MAX_HDRS_LEN);
 	  if (peek_data)
 	    {
diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index 86bce4895b6..07f183891ea 100644
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -1713,6 +1713,14 @@ tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  tcp_connection_t *tc0;
 	  u32 next0 = tcp_next_drop (is_ip4), error0 = TCP_ERROR_ENQUEUED;
 
+	  if (n_left_from > 1)
+	    {
+	      vlib_buffer_t *pb;
+	      pb = vlib_get_buffer (vm, from[1]);
+	      vlib_prefetch_buffer_header (pb, LOAD);
+	      CLIB_PREFETCH (pb->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
+	    }
+
 	  bi0 = from[0];
 	  to_next[0] = bi0;
 	  from += 1;
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index bbcbc912175..2b5a5cd73fe 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -1771,6 +1771,14 @@ tcp46_output_inline (vlib_main_t * vm,
 	  tcp_header_t *th0 = 0;
 	  u32 error0 = TCP_ERROR_PKTS_SENT, next0 = TCP_OUTPUT_NEXT_IP_LOOKUP;
 
+	  if (n_left_from > 1)
+	    {
+	      vlib_buffer_t *pb;
+	      pb = vlib_get_buffer (vm, from[1]);
+	      vlib_prefetch_buffer_header (pb, STORE);
+	      CLIB_PREFETCH (pb->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
+	    }
+
 	  bi0 = from[0];
 	  to_next[0] = bi0;
 	  from += 1;