diff options
| -rw-r--r-- | src/vnet/pg/init.c | 4 | ||||
| -rw-r--r-- | src/vnet/pg/input.c | 229 | ||||
| -rw-r--r-- | src/vnet/pg/pg.h | 3 |
3 files changed, 165 insertions, 71 deletions
diff --git a/src/vnet/pg/init.c b/src/vnet/pg/init.c index 631be25ea3c..5db9c00ffc4 100644 --- a/src/vnet/pg/init.c +++ b/src/vnet/pg/init.c @@ -48,6 +48,8 @@ pg_init (vlib_main_t * vm) { clib_error_t *error; pg_main_t *pg = &pg_main; + vlib_thread_main_t *vtm = vlib_get_thread_main (); + u32 num_threads = 1 /* main thread */ + vtm->n_threads; pg->if_index_by_if_id = hash_create (0, sizeof (uword)); @@ -57,6 +59,8 @@ pg_init (vlib_main_t * vm) if ((error = vlib_call_init_function (vm, pg_cli_init))) goto done; + vec_validate (pg->replay_buffers_by_thread, num_threads); + done: return error; } diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index 4db4b45edcb..792c67cd88d 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -37,6 +37,14 @@ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ + /* + * To be honest, the packet generator needs an extreme + * makeover. Two key assumptions which drove the current implementation + * are no longer true. First, buffer managers implement a + * post-TX recycle list. Second, that packet generator performance + * is first-order important. + */ + #include <vlib/vlib.h> #include <vnet/pg/pg.h> #include <vnet/vnet.h> @@ -1055,49 +1063,6 @@ pg_set_next_buffer_pointers (pg_main_t * pg, } static_always_inline void -init_replay_buffers_inline (vlib_main_t * vm, - pg_stream_t * s, - u32 * buffers, - u32 n_buffers, u32 data_offset, u32 n_data) -{ - u32 n_left, *b, i, l; - - n_left = n_buffers; - b = buffers; - i = s->current_replay_packet_index; - l = vec_len (s->replay_packet_templates); - - while (n_left >= 1) - { - u32 bi0, n0; - vlib_buffer_t *b0; - u8 *d0; - - bi0 = b[0]; - b += 1; - n_left -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - vnet_buffer (b0)->sw_if_index[VLIB_RX] = s->sw_if_index[VLIB_RX]; - /* was s->sw_if_index[VLIB_TX]; */ - vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; - - d0 = vec_elt (s->replay_packet_templates, i); - vnet_buffer2 (b0)->pg_replay_timestamp = s->replay_packet_timestamps[i]; - - n0 = n_data; - if (data_offset + n_data >= vec_len (d0)) - n0 = vec_len (d0) > data_offset ? vec_len (d0) - data_offset : 0; - - b0->current_length = n0; - - clib_memcpy_fast (b0->data, d0 + data_offset, n0); - i = i + 1 == l ? 0 : i + 1; - } -} - -static_always_inline void init_buffers_inline (vlib_main_t * vm, pg_stream_t * s, u32 * buffers, @@ -1106,9 +1071,7 @@ init_buffers_inline (vlib_main_t * vm, u32 n_left, *b; u8 *data, *mask; - if (vec_len (s->replay_packet_templates) > 0) - return init_replay_buffers_inline (vm, s, buffers, n_buffers, data_offset, - n_data); + ASSERT (s->replay_packet_templates == 0); data = s->fixed_packet_data + data_offset; mask = s->fixed_packet_data_mask + data_offset; @@ -1190,6 +1153,8 @@ pg_stream_fill_helper (pg_main_t * pg, uword is_start_of_packet = bi == s->buffer_indices; u32 n_allocated; + ASSERT (vec_len (s->replay_packet_templates) == 0); + n_allocated = vlib_buffer_alloc (vm, buffers, n_alloc); if (n_allocated == 0) return 0; @@ -1213,37 +1178,134 @@ pg_stream_fill_helper (pg_main_t * pg, if (is_start_of_packet) { - if (vec_len (s->replay_packet_templates) > 0) + pg_generate_set_lengths (pg, s, buffers, n_alloc); + if (vec_len (s->buffer_indices) > 1) + pg_generate_fix_multi_buffer_lengths (pg, s, buffers, n_alloc); + + pg_generate_edit (pg, s, buffers, n_alloc); + } + + return n_alloc; +} + +static u32 +pg_stream_fill_replay (pg_main_t * pg, pg_stream_t * s, u32 n_alloc) +{ + pg_buffer_index_t *bi; + u32 n_left, i, l; + u32 buffer_alloc_request = 0; + u32 buffer_alloc_result; + u32 current_buffer_index; + u32 *buffers; + vlib_main_t *vm = vlib_get_main (); + vnet_main_t *vnm = vnet_get_main (); + vnet_interface_main_t *im = &vnm->interface_main; + vnet_sw_interface_t *si; + + buffers = pg->replay_buffers_by_thread[vm->thread_index]; + vec_reset_length (buffers); + bi = s->buffer_indices; + + n_left = n_alloc; + i = s->current_replay_packet_index; + l = vec_len (s->replay_packet_templates); + + /* Figure out how many buffers we need */ + while (n_left > 0) + { + u8 *d0; + + d0 = vec_elt (s->replay_packet_templates, i); + buffer_alloc_request += (vec_len (d0) + (VLIB_BUFFER_DATA_SIZE - 1)) + / VLIB_BUFFER_DATA_SIZE; + + i = ((i + 1) == l) ? 0 : i + 1; + n_left--; + } + + ASSERT (buffer_alloc_request > 0); + vec_validate (buffers, buffer_alloc_request - 1); + + /* Allocate that many buffers */ + buffer_alloc_result = vlib_buffer_alloc (vm, buffers, buffer_alloc_request); + if (buffer_alloc_result < buffer_alloc_request) + { + clib_warning ("alloc failure, got %d not %d", buffer_alloc_result, + buffer_alloc_request); + vlib_buffer_free_no_next (vm, buffers, buffer_alloc_result); + pg->replay_buffers_by_thread[vm->thread_index] = buffers; + return 0; + } + + /* Now go generate the buffers, and add them to the FIFO */ + n_left = n_alloc; + + current_buffer_index = 0; + i = s->current_replay_packet_index; + l = vec_len (s->replay_packet_templates); + while (n_left > 0) + { + u8 *d0; + int not_last; + u32 data_offset; + u32 bytes_to_copy, bytes_this_chunk; + vlib_buffer_t *b; + + d0 = vec_elt (s->replay_packet_templates, i); + data_offset = 0; + bytes_to_copy = vec_len (d0); + + /* Add head chunk to pg fifo */ + clib_fifo_add1 (bi->buffer_fifo, buffers[current_buffer_index]); + + /* Copy the data */ + while (bytes_to_copy) { - vnet_main_t *vnm = vnet_get_main (); - vnet_interface_main_t *im = &vnm->interface_main; - vnet_sw_interface_t *si = - vnet_get_sw_interface (vnm, s->sw_if_index[VLIB_RX]); - u32 l = 0; - u32 i; - for (i = 0; i < n_alloc; i++) - l += vlib_buffer_index_length_in_chain (vm, buffers[i]); - vlib_increment_combined_counter (im->combined_sw_if_counters - + VNET_INTERFACE_COUNTER_RX, - vlib_get_thread_index (), - si->sw_if_index, n_alloc, l); - s->current_replay_packet_index += n_alloc; - s->current_replay_packet_index %= - vec_len (s->replay_packet_templates); + bytes_this_chunk = clib_min (bytes_to_copy, VLIB_BUFFER_DATA_SIZE); + ASSERT (current_buffer_index < vec_len (buffers)); + b = vlib_get_buffer (vm, buffers[current_buffer_index]); + clib_memcpy_fast (b->data, d0 + data_offset, bytes_this_chunk); + vnet_buffer (b)->sw_if_index[VLIB_RX] = s->sw_if_index[VLIB_RX]; + vnet_buffer (b)->sw_if_index[VLIB_TX] = (u32) ~ 0; + b->flags = 0; + b->next_buffer = 0; + b->current_data = 0; + b->current_length = bytes_this_chunk; + + not_last = bytes_this_chunk < bytes_to_copy; + if (not_last) + { + ASSERT (current_buffer_index < (vec_len (buffers) - 1)); + b->flags |= VLIB_BUFFER_NEXT_PRESENT; + b->next_buffer = buffers[current_buffer_index + 1]; + } + bytes_to_copy -= bytes_this_chunk; + data_offset += bytes_this_chunk; + current_buffer_index++; } - else - { - pg_generate_set_lengths (pg, s, buffers, n_alloc); - if (vec_len (s->buffer_indices) > 1) - pg_generate_fix_multi_buffer_lengths (pg, s, buffers, n_alloc); - pg_generate_edit (pg, s, buffers, n_alloc); - } + i = ((i + 1) == l) ? 0 : i + 1; + n_left--; } + /* Update the interface counters */ + si = vnet_get_sw_interface (vnm, s->sw_if_index[VLIB_RX]); + l = 0; + for (i = 0; i < n_alloc; i++) + l += vlib_buffer_index_length_in_chain (vm, buffers[i]); + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + vlib_get_thread_index (), + si->sw_if_index, n_alloc, l); + + s->current_replay_packet_index += n_alloc; + s->current_replay_packet_index %= vec_len (s->replay_packet_templates); + + pg->replay_buffers_by_thread[vm->thread_index] = buffers; return n_alloc; } + static u32 pg_stream_fill (pg_main_t * pg, pg_stream_t * s, u32 n_buffers) { @@ -1270,6 +1332,12 @@ pg_stream_fill (pg_main_t * pg, pg_stream_t * s, u32 n_buffers) n_alloc = 0; } + /* + * Handle pcap replay directly + */ + if (s->replay_packet_templates) + return pg_stream_fill_replay (pg, s, n_alloc); + /* All buffer fifos should have the same size. */ if (CLIB_DEBUG > 0) { @@ -1531,8 +1599,15 @@ pg_generate_packets (vlib_node_runtime_t * node, (n_this_frame - n) * sizeof (u32)); } - vec_foreach (bi, s->buffer_indices) - clib_fifo_advance_head (bi->buffer_fifo, n_this_frame); + if (s->replay_packet_templates == 0) + { + vec_foreach (bi, s->buffer_indices) + clib_fifo_advance_head (bi->buffer_fifo, n_this_frame); + } + else + { + clib_fifo_advance_head (bi0->buffer_fifo, n_this_frame); + } if (current_config_index != ~(u32) 0) for (i = 0; i < n_this_frame; i++) @@ -1553,6 +1628,18 @@ pg_generate_packets (vlib_node_runtime_t * node, n_packets_to_generate -= n_this_frame; n_packets_generated += n_this_frame; n_left -= n_this_frame; + if (CLIB_DEBUG > 0) + { + int i; + vlib_buffer_t *b; + + for (i = 0; i < VLIB_FRAME_SIZE - n_left; i++) + { + b = vlib_get_buffer (vm, to_next[i]); + ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0 || + b->current_length >= VLIB_BUFFER_MIN_CHAIN_SEG_SIZE); + } + } vlib_put_next_frame (vm, node, next_index, n_left); } diff --git a/src/vnet/pg/pg.h b/src/vnet/pg/pg.h index 0b06803d384..16463c2a8c5 100644 --- a/src/vnet/pg/pg.h +++ b/src/vnet/pg/pg.h @@ -326,6 +326,9 @@ typedef struct pg_main_t pg_interface_t *interfaces; uword *if_index_by_if_id; + /* Vector of buffer indices for use in pg_stream_fill_replay, per thread */ + u32 **replay_buffers_by_thread; + /* Per VLIB node information. */ pg_node_t *nodes; } pg_main_t; |