diff options
| -rw-r--r-- | src/vnet/session/session.h | 1 | ||||
| -rw-r--r-- | src/vnet/session/session_node.c | 16 | ||||
| -rw-r--r-- | src/vnet/session/transport.h | 3 | ||||
| -rw-r--r-- | src/vnet/tcp/tcp.h | 4 | ||||
| -rw-r--r-- | src/vnet/tcp/tcp_output.c | 37 | ||||
| -rw-r--r-- | src/vnet/udp/udp.c | 44 |
6 files changed, 83 insertions, 22 deletions
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index e549da588e9..9d6c945bb4d 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -59,6 +59,7 @@ typedef struct session_tx_context_ /** Vector of tx buffer free lists */ u32 *tx_buffers; + vlib_buffer_t **transport_pending_bufs; } session_tx_context_t; typedef struct session_evt_elt diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c index bebabd3d68e..403c28f28ce 100644 --- a/src/vnet/session/session_node.c +++ b/src/vnet/session/session_node.c @@ -1319,6 +1319,8 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk, ctx->left_to_snd = ctx->max_len_to_snd; n_left = ctx->n_segs_per_evt; + vec_validate (ctx->transport_pending_bufs, n_left); + while (n_left >= 4) { vlib_buffer_t *b0, *b1; @@ -1340,9 +1342,8 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk, session_tx_fill_buffer (vm, ctx, b0, &n_bufs, peek_data); session_tx_fill_buffer (vm, ctx, b1, &n_bufs, peek_data); - ctx->transport_vft->push_header (ctx->tc, b0); - ctx->transport_vft->push_header (ctx->tc, b1); - + ctx->transport_pending_bufs[ctx->n_segs_per_evt - n_left] = b0; + ctx->transport_pending_bufs[ctx->n_segs_per_evt - n_left + 1] = b1; n_left -= 2; vec_add1 (wrk->pending_tx_buffers, bi0); @@ -1366,16 +1367,17 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk, b0 = vlib_get_buffer (vm, bi0); session_tx_fill_buffer (vm, ctx, b0, &n_bufs, peek_data); - /* Ask transport to push header after current_length and - * total_length_not_including_first_buffer are updated */ - ctx->transport_vft->push_header (ctx->tc, b0); - + ctx->transport_pending_bufs[ctx->n_segs_per_evt - n_left] = b0; n_left -= 1; vec_add1 (wrk->pending_tx_buffers, bi0); vec_add1 (wrk->pending_tx_nexts, next_index); } + /* Ask transport to push headers */ + ctx->transport_vft->push_header (ctx->tc, ctx->transport_pending_bufs, + ctx->n_segs_per_evt); + if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node)) > 0)) session_tx_trace_frame (vm, node, next_index, wrk->pending_tx_buffers, ctx->n_segs_per_evt, ctx->s, n_trace); diff --git a/src/vnet/session/transport.h b/src/vnet/session/transport.h index 549f17957cc..4edc281ff9c 100644 --- a/src/vnet/session/transport.h +++ b/src/vnet/session/transport.h @@ -85,7 +85,8 @@ typedef struct _transport_proto_vft * Transmission */ - u32 (*push_header) (transport_connection_t * tconn, vlib_buffer_t * b); + u32 (*push_header) (transport_connection_t *tconn, vlib_buffer_t **b, + u32 n_bufs); int (*send_params) (transport_connection_t * tconn, transport_send_params_t *sp); void (*update_time) (f64 time_now, u8 thread_index); diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h index 710561fb24a..c5dd3172302 100644 --- a/src/vnet/tcp/tcp.h +++ b/src/vnet/tcp/tcp.h @@ -313,8 +313,8 @@ u32 tcp_snd_space (tcp_connection_t * tc); int tcp_fastrecovery_prr_snd_space (tcp_connection_t * tc); void tcp_reschedule (tcp_connection_t * tc); fib_node_index_t tcp_lookup_rmt_in_fib (tcp_connection_t * tc); -u32 tcp_session_push_header (transport_connection_t * tconn, - vlib_buffer_t * b); +u32 tcp_session_push_header (transport_connection_t *tconn, vlib_buffer_t **b, + u32 n_bufs); int tcp_session_custom_tx (void *conn, transport_send_params_t * sp); void tcp_connection_timers_init (tcp_connection_t * tc); diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index 55fa6d1b5d8..88cd91330b0 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -963,11 +963,9 @@ tcp_buffer_len (vlib_buffer_t * b) return data_len; } -u32 -tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b) +always_inline u32 +tcp_push_one_header (tcp_connection_t *tc, vlib_buffer_t *b) { - tcp_connection_t *tc = (tcp_connection_t *) tconn; - if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE) tcp_bt_track_tx (tc, tcp_buffer_len (b)); @@ -975,6 +973,37 @@ tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b) /* update_snd_nxt */ 1); tcp_validate_txf_size (tc, tc->snd_nxt - tc->snd_una); + return 0; +} + +u32 +tcp_session_push_header (transport_connection_t *tconn, vlib_buffer_t **bs, + u32 n_bufs) +{ + tcp_connection_t *tc = (tcp_connection_t *) tconn; + + while (n_bufs >= 4) + { + vlib_prefetch_buffer_header (bs[2], STORE); + vlib_prefetch_buffer_header (bs[3], STORE); + + tcp_push_one_header (tc, bs[0]); + tcp_push_one_header (tc, bs[1]); + + n_bufs -= 2; + bs += 2; + } + while (n_bufs) + { + if (n_bufs > 1) + vlib_prefetch_buffer_header (bs[1], STORE); + + tcp_push_one_header (tc, bs[0]); + + n_bufs -= 1; + bs += 1; + } + /* If not tracking an ACK, start tracking */ if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc)) { diff --git a/src/vnet/udp/udp.c b/src/vnet/udp/udp.c index 260ea9b6287..5f73e3968c0 100644 --- a/src/vnet/udp/udp.c +++ b/src/vnet/udp/udp.c @@ -236,16 +236,11 @@ udp_session_get_listener (u32 listener_index) return &us->connection; } -static u32 -udp_push_header (transport_connection_t * tc, vlib_buffer_t * b) +always_inline u32 +udp_push_one_header (vlib_main_t *vm, udp_connection_t *uc, vlib_buffer_t *b) { - udp_connection_t *uc; - vlib_main_t *vm = vlib_get_main (); - - uc = udp_connection_from_transport (tc); - vlib_buffer_push_udp (b, uc->c_lcl_port, uc->c_rmt_port, 1); - if (tc->is_ip4) + if (uc->c_is_ip4) vlib_buffer_push_ip4_custom (vm, b, &uc->c_lcl_ip4, &uc->c_rmt_ip4, IP_PROTOCOL_UDP, 1 /* csum offload */, 0 /* is_df */, uc->c_dscp); @@ -256,6 +251,39 @@ udp_push_header (transport_connection_t * tc, vlib_buffer_t * b) vnet_buffer (b)->sw_if_index[VLIB_TX] = uc->c_fib_index; b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; + return 0; +} + +static u32 +udp_push_header (transport_connection_t *tc, vlib_buffer_t **bs, u32 n_bufs) +{ + vlib_main_t *vm = vlib_get_main (); + udp_connection_t *uc; + + uc = udp_connection_from_transport (tc); + + while (n_bufs >= 4) + { + vlib_prefetch_buffer_header (bs[2], STORE); + vlib_prefetch_buffer_header (bs[3], STORE); + + udp_push_one_header (vm, uc, bs[0]); + udp_push_one_header (vm, uc, bs[1]); + + n_bufs -= 2; + bs += 2; + } + while (n_bufs) + { + if (n_bufs > 1) + vlib_prefetch_buffer_header (bs[1], STORE); + + udp_push_one_header (vm, uc, bs[0]); + + n_bufs -= 1; + bs += 1; + } + if (PREDICT_FALSE (uc->flags & UDP_CONN_F_CLOSING)) { if (!transport_max_tx_dequeue (&uc->connection)) |