session: pass tx buffers in bulk to transports

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I1025cccd784f80b557847f69c3ea1ada5c9de60d

6 files changed, 83 insertions, 22 deletions

diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h
index e549da588e9..9d6c945bb4d 100644
--- a/src/vnet/session/session.h
+++ b/src/vnet/session/session.h
@@ -59,6 +59,7 @@ typedef struct session_tx_context_
 
   /** Vector of tx buffer free lists */
   u32 *tx_buffers;
+  vlib_buffer_t **transport_pending_bufs;
 } session_tx_context_t;
 
 typedef struct session_evt_elt
diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index bebabd3d68e..403c28f28ce 100644
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -1319,6 +1319,8 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk,
   ctx->left_to_snd = ctx->max_len_to_snd;
   n_left = ctx->n_segs_per_evt;
 
+  vec_validate (ctx->transport_pending_bufs, n_left);
+
   while (n_left >= 4)
     {
       vlib_buffer_t *b0, *b1;
@@ -1340,9 +1342,8 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk,
       session_tx_fill_buffer (vm, ctx, b0, &n_bufs, peek_data);
       session_tx_fill_buffer (vm, ctx, b1, &n_bufs, peek_data);
 
-      ctx->transport_vft->push_header (ctx->tc, b0);
-      ctx->transport_vft->push_header (ctx->tc, b1);
-
+      ctx->transport_pending_bufs[ctx->n_segs_per_evt - n_left] = b0;
+      ctx->transport_pending_bufs[ctx->n_segs_per_evt - n_left + 1] = b1;
       n_left -= 2;
 
       vec_add1 (wrk->pending_tx_buffers, bi0);
@@ -1366,16 +1367,17 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk,
       b0 = vlib_get_buffer (vm, bi0);
       session_tx_fill_buffer (vm, ctx, b0, &n_bufs, peek_data);
 
-      /* Ask transport to push header after current_length and
-       * total_length_not_including_first_buffer are updated */
-      ctx->transport_vft->push_header (ctx->tc, b0);
-
+      ctx->transport_pending_bufs[ctx->n_segs_per_evt - n_left] = b0;
       n_left -= 1;
 
       vec_add1 (wrk->pending_tx_buffers, bi0);
       vec_add1 (wrk->pending_tx_nexts, next_index);
     }
 
+  /* Ask transport to push headers */
+  ctx->transport_vft->push_header (ctx->tc, ctx->transport_pending_bufs,
+				   ctx->n_segs_per_evt);
+
   if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node)) > 0))
     session_tx_trace_frame (vm, node, next_index, wrk->pending_tx_buffers,
 			    ctx->n_segs_per_evt, ctx->s, n_trace);
diff --git a/src/vnet/session/transport.h b/src/vnet/session/transport.h
index 549f17957cc..4edc281ff9c 100644
--- a/src/vnet/session/transport.h
+++ b/src/vnet/session/transport.h
@@ -85,7 +85,8 @@ typedef struct _transport_proto_vft
    * Transmission
    */
 
-  u32 (*push_header) (transport_connection_t * tconn, vlib_buffer_t * b);
+  u32 (*push_header) (transport_connection_t *tconn, vlib_buffer_t **b,
+		      u32 n_bufs);
   int (*send_params) (transport_connection_t * tconn,
 		      transport_send_params_t *sp);
   void (*update_time) (f64 time_now, u8 thread_index);
diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h
index 710561fb24a..c5dd3172302 100644
--- a/src/vnet/tcp/tcp.h
+++ b/src/vnet/tcp/tcp.h
@@ -313,8 +313,8 @@ u32 tcp_snd_space (tcp_connection_t * tc);
 int tcp_fastrecovery_prr_snd_space (tcp_connection_t * tc);
 void tcp_reschedule (tcp_connection_t * tc);
 fib_node_index_t tcp_lookup_rmt_in_fib (tcp_connection_t * tc);
-u32 tcp_session_push_header (transport_connection_t * tconn,
-			     vlib_buffer_t * b);
+u32 tcp_session_push_header (transport_connection_t *tconn, vlib_buffer_t **b,
+			     u32 n_bufs);
 int tcp_session_custom_tx (void *conn, transport_send_params_t * sp);
 
 void tcp_connection_timers_init (tcp_connection_t * tc);
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index 55fa6d1b5d8..88cd91330b0 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -963,11 +963,9 @@ tcp_buffer_len (vlib_buffer_t * b)
   return data_len;
 }
 
-u32
-tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
+always_inline u32
+tcp_push_one_header (tcp_connection_t *tc, vlib_buffer_t *b)
 {
-  tcp_connection_t *tc = (tcp_connection_t *) tconn;
-
   if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
     tcp_bt_track_tx (tc, tcp_buffer_len (b));
 
@@ -975,6 +973,37 @@ tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
 		  /* update_snd_nxt */ 1);
 
   tcp_validate_txf_size (tc, tc->snd_nxt - tc->snd_una);
+  return 0;
+}
+
+u32
+tcp_session_push_header (transport_connection_t *tconn, vlib_buffer_t **bs,
+			 u32 n_bufs)
+{
+  tcp_connection_t *tc = (tcp_connection_t *) tconn;
+
+  while (n_bufs >= 4)
+    {
+      vlib_prefetch_buffer_header (bs[2], STORE);
+      vlib_prefetch_buffer_header (bs[3], STORE);
+
+      tcp_push_one_header (tc, bs[0]);
+      tcp_push_one_header (tc, bs[1]);
+
+      n_bufs -= 2;
+      bs += 2;
+    }
+  while (n_bufs)
+    {
+      if (n_bufs > 1)
+	vlib_prefetch_buffer_header (bs[1], STORE);
+
+      tcp_push_one_header (tc, bs[0]);
+
+      n_bufs -= 1;
+      bs += 1;
+    }
+
   /* If not tracking an ACK, start tracking */
   if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
     {
diff --git a/src/vnet/udp/udp.c b/src/vnet/udp/udp.c
index 260ea9b6287..5f73e3968c0 100644
--- a/src/vnet/udp/udp.c
+++ b/src/vnet/udp/udp.c
@@ -236,16 +236,11 @@ udp_session_get_listener (u32 listener_index)
   return &us->connection;
 }
 
-static u32
-udp_push_header (transport_connection_t * tc, vlib_buffer_t * b)
+always_inline u32
+udp_push_one_header (vlib_main_t *vm, udp_connection_t *uc, vlib_buffer_t *b)
 {
-  udp_connection_t *uc;
-  vlib_main_t *vm = vlib_get_main ();
-
-  uc = udp_connection_from_transport (tc);
-
   vlib_buffer_push_udp (b, uc->c_lcl_port, uc->c_rmt_port, 1);
-  if (tc->is_ip4)
+  if (uc->c_is_ip4)
     vlib_buffer_push_ip4_custom (vm, b, &uc->c_lcl_ip4, &uc->c_rmt_ip4,
 				 IP_PROTOCOL_UDP, 1 /* csum offload */,
 				 0 /* is_df */, uc->c_dscp);
@@ -256,6 +251,39 @@ udp_push_header (transport_connection_t * tc, vlib_buffer_t * b)
   vnet_buffer (b)->sw_if_index[VLIB_TX] = uc->c_fib_index;
   b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
 
+  return 0;
+}
+
+static u32
+udp_push_header (transport_connection_t *tc, vlib_buffer_t **bs, u32 n_bufs)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  udp_connection_t *uc;
+
+  uc = udp_connection_from_transport (tc);
+
+  while (n_bufs >= 4)
+    {
+      vlib_prefetch_buffer_header (bs[2], STORE);
+      vlib_prefetch_buffer_header (bs[3], STORE);
+
+      udp_push_one_header (vm, uc, bs[0]);
+      udp_push_one_header (vm, uc, bs[1]);
+
+      n_bufs -= 2;
+      bs += 2;
+    }
+  while (n_bufs)
+    {
+      if (n_bufs > 1)
+	vlib_prefetch_buffer_header (bs[1], STORE);
+
+      udp_push_one_header (vm, uc, bs[0]);
+
+      n_bufs -= 1;
+      bs += 1;
+    }
+
   if (PREDICT_FALSE (uc->flags & UDP_CONN_F_CLOSING))
     {
       if (!transport_max_tx_dequeue (&uc->connection))