From 7cd468a3d7dee7d6c92f69a0bb7061ae208ec727 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 19 Dec 2016 23:05:39 +0100 Subject: Reorganize source tree to use single autotools instance Change-Id: I7b51f88292e057c6443b12224486f2d0c9f8ae23 Signed-off-by: Damjan Marion --- src/vnet/unix/gdb_funcs.c | 171 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 src/vnet/unix/gdb_funcs.c (limited to 'src/vnet/unix/gdb_funcs.c') diff --git a/src/vnet/unix/gdb_funcs.c b/src/vnet/unix/gdb_funcs.c new file mode 100644 index 00000000..cfb4b247 --- /dev/null +++ b/src/vnet/unix/gdb_funcs.c @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @file + * @brief Host utility functions + */ +#include +#include + +#include + + + +/** + * @brief GDB callable function: vl - Return vector length of vector + * + * @param *p - void - address of vector + * + * @return length - u32 + * + */ +u32 vl(void *p) +{ + return vec_len (p); +} + +/** + * @brief GDB callable function: pe - call pool_elts - number of elements in a pool + * + * @param *v - void - address of pool + * + * @return number - uword + * + */ +uword pe (void *v) +{ + return (pool_elts(v)); +} + +/** + * @brief GDB callable function: pifi - call pool_is_free_index - is passed index free? + * + * @param *p - void - address of pool + * @param *index - u32 + * + * @return 0|1 - int + * + */ +int pifi (void *p, u32 index) +{ + return pool_is_free_index (p, index); +} + +/** + * @brief GDB callable function: debug_hex_bytes - return formatted hex string + * + * @param *s - u8 + * @param n - u32 - number of bytes to format + * + */ +void debug_hex_bytes (u8 *s, u32 n) +{ + fformat (stderr, "%U\n", format_hex_bytes, s, n); +} + +/** + * @brief GDB callable function: vlib_dump_frame_ownership + * + */ +void vlib_dump_frame_ownership (void) +{ + vlib_main_t * vm = vlib_get_main(); + vlib_node_main_t * nm = &vm->node_main; + vlib_node_runtime_t * this_node_runtime; + vlib_next_frame_t * nf; + u32 first_nf_index; + u32 index; + + vec_foreach(this_node_runtime, nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) + { + first_nf_index = this_node_runtime->next_frame_index; + + for (index = first_nf_index; index < first_nf_index + + this_node_runtime->n_next_nodes; index++) + { + vlib_node_runtime_t * owned_runtime; + nf = vec_elt_at_index (vm->node_main.next_frames, index); + if (nf->flags & VLIB_FRAME_OWNER) + { + owned_runtime = vec_elt_at_index (nm->nodes_by_type[0], + nf->node_runtime_index); + fformat(stderr, + "%s next index %d owns enqueue rights to %s\n", + nm->nodes[this_node_runtime->node_index]->name, + index - first_nf_index, + nm->nodes[owned_runtime->node_index]->name); + fformat (stderr, " nf index %d nf->frame_index %d\n", + nf - vm->node_main.next_frames, + nf->frame_index); + } + } + } +} + +/** + * @brief GDB callable function: vlib_runtime_index_to_node_name + * + * Takes node index and will return the node name. + * + * @param index - u32 + */ +void vlib_runtime_index_to_node_name (u32 index) +{ + vlib_main_t * vm = vlib_get_main(); + vlib_node_main_t * nm = &vm->node_main; + + if (index > vec_len (nm->nodes)) + { + fformat(stderr, "%d out of range, max %d\n", vec_len(nm->nodes)); + return; + } + + fformat(stderr, "node runtime index %d name %s\n", index, nm->nodes[index]->name); +} + + +/** + * @brief GDB callable function: show_gdb_command_fn - show gdb + * + * Shows list of functions for VPP available in GDB + * + * @return error - clib_error_t + */ +static clib_error_t * +show_gdb_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + vlib_cli_output (vm, "vl(p) returns vec_len(p)"); + vlib_cli_output (vm, "pe(p) returns pool_elts(p)"); + vlib_cli_output (vm, "pifi(p, i) returns pool_is_free_index(p, i)"); + vlib_cli_output (vm, "debug_hex_bytes (ptr, n_bytes) dumps n_bytes in hex"); + vlib_cli_output (vm, "vlib_dump_frame_ownership() does what it says"); + vlib_cli_output (vm, "vlib_runtime_index_to_node_name (index) prints NN"); + + return 0; +} + +VLIB_CLI_COMMAND (show_gdb_funcs_command, static) = { + .path = "show gdb", + .short_help = "Describe functions which can be called from gdb", + .function = show_gdb_command_fn, +}; + +/* Cafeteria plan, maybe you don't want these functions */ +clib_error_t * +gdb_func_init (vlib_main_t * vm) { return 0; } + +VLIB_INIT_FUNCTION (gdb_func_init); -- cgit 1.2.3-korg From acd2a6a32cebab70a054e9335e446798538d0524 Mon Sep 17 00:00:00 2001 From: Dave Barach Date: Tue, 16 May 2017 17:41:34 -0400 Subject: VPP-846: tcp perf / scale / hardening Fix builtin server event queue handling Change-Id: I21b49c37188746cadb2fd9423291c5dc1335798c Signed-off-by: Dave Barach --- src/svm/svm_fifo.h | 4 +- src/vnet/session/node.c | 108 ++++++++++++++++++++++++++++++------------ src/vnet/session/session.c | 16 ++++--- src/vnet/session/session.h | 4 +- src/vnet/tcp/builtin_client.c | 2 +- src/vnet/tcp/builtin_server.c | 12 ++--- src/vnet/unix/gdb_funcs.c | 12 ++++- 7 files changed, 108 insertions(+), 50 deletions(-) (limited to 'src/vnet/unix/gdb_funcs.c') diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index 36158dc5..69369163 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -44,7 +44,7 @@ typedef struct u32 nitems; CLIB_CACHE_LINE_ALIGN_MARK (end_cursize); - volatile u8 has_event; /**< non-zero if deq event exists */ + volatile u32 has_event; /**< non-zero if deq event exists */ /* Backpointers */ u32 master_session_index; @@ -103,7 +103,7 @@ always_inline void svm_fifo_unset_event (svm_fifo_t * f) { /* Probably doesn't need to be atomic. Still, better avoid surprises */ - __sync_lock_test_and_set (&f->has_event, 0); + __sync_lock_release (&f->has_event); } svm_fifo_t *svm_fifo_create (u32 data_size_in_bytes); diff --git a/src/vnet/session/node.c b/src/vnet/session/node.c index fffc8eb3..3053ccc2 100644 --- a/src/vnet/session/node.c +++ b/src/vnet/session/node.c @@ -154,7 +154,7 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node, /* Can't make any progress */ if (snd_space0 == 0 || snd_mss0 == 0) { - vec_add1 (smm->evts_partially_read[thread_index], *e0); + vec_add1 (smm->pending_event_vector[thread_index], *e0); return 0; } @@ -216,7 +216,7 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node, { if (svm_fifo_set_event (s0->server_tx_fifo)) { - vec_add1 (smm->evts_partially_read[thread_index], *e0); + vec_add1 (smm->pending_event_vector[thread_index], *e0); } return -1; } @@ -324,7 +324,7 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node, /* If we don't already have new event */ if (svm_fifo_set_event (s0->server_tx_fifo)) { - vec_add1 (smm->evts_partially_read[thread_index], *e0); + vec_add1 (smm->pending_event_vector[thread_index], *e0); } } return 0; @@ -338,7 +338,7 @@ dequeue_fail: if (svm_fifo_set_event (s0->server_tx_fifo)) { - vec_add1 (smm->evts_partially_read[thread_index], *e0); + vec_add1 (smm->pending_event_vector[thread_index], *e0); } vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); _vec_len (smm->tx_buffers[thread_index]) += 1; @@ -388,12 +388,70 @@ session_event_get_session (session_fifo_event_t * e0, u8 thread_index) return s0; } +void +dump_thread_0_event_queue (void) +{ + session_manager_main_t *smm = vnet_get_session_manager_main (); + vlib_main_t *vm = &vlib_global_main; + u32 my_thread_index = vm->thread_index; + session_fifo_event_t _e, *e = &_e; + stream_session_t *s0; + int i, index; + i8 *headp; + + unix_shared_memory_queue_t *q; + q = smm->vpp_event_queues[my_thread_index]; + + index = q->head; + + for (i = 0; i < q->cursize; i++) + { + headp = (i8 *) (&q->data[0] + q->elsize * index); + clib_memcpy (e, headp, q->elsize); + + switch (e->event_type) + { + case FIFO_EVENT_APP_TX: + s0 = session_event_get_session (e, my_thread_index); + fformat (stdout, "[%04d] TX session %d\n", i, s0->session_index); + break; + + case FIFO_EVENT_DISCONNECT: + s0 = stream_session_get_from_handle (e->session_handle); + fformat (stdout, "[%04d] disconnect session %d\n", i, + s0->session_index); + break; + + case FIFO_EVENT_BUILTIN_RX: + s0 = session_event_get_session (e, my_thread_index); + fformat (stdout, "[%04d] builtin_rx %d\n", i, s0->session_index); + break; + + case FIFO_EVENT_RPC: + fformat (stdout, "[%04d] RPC call %llx with %llx\n", + i, (u64) (e->rpc_args.fp), (u64) (e->rpc_args.arg)); + break; + + default: + fformat (stdout, "[%04d] unhandled event type %d\n", + i, e->event_type); + break; + } + + index++; + + if (index == q->maxsize) + index = 0; + } +} + static uword session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { session_manager_main_t *smm = vnet_get_session_manager_main (); - session_fifo_event_t *my_fifo_events, *e; + session_fifo_event_t *my_pending_event_vector, *e; + session_fifo_event_t *my_fifo_events; u32 n_to_dequeue, n_events; unix_shared_memory_queue_t *q; application_t *app; @@ -417,11 +475,13 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, if (PREDICT_FALSE (q == 0)) return 0; + my_fifo_events = smm->free_event_vector[my_thread_index]; + /* min number of events we can dequeue without blocking */ n_to_dequeue = q->cursize; - my_fifo_events = smm->fifo_events[my_thread_index]; + my_pending_event_vector = smm->pending_event_vector[my_thread_index]; - if (n_to_dequeue == 0 && vec_len (my_fifo_events) == 0) + if (n_to_dequeue == 0 && vec_len (my_pending_event_vector) == 0) return 0; SESSION_EVT_DBG (SESSION_EVT_DEQ_NODE, 0); @@ -431,7 +491,7 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, * over them again without dequeuing new ones. */ /* XXX: Block senders to sessions that can't keep up */ - if (vec_len (my_fifo_events) >= 100) + if (0 && vec_len (my_pending_event_vector) >= 100) { clib_warning ("too many fifo events unsolved"); goto skip_dequeue; @@ -452,7 +512,10 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, (void) pthread_cond_broadcast (&q->condvar); pthread_mutex_unlock (&q->mutex); - smm->fifo_events[my_thread_index] = my_fifo_events; + vec_append (my_fifo_events, my_pending_event_vector); + + _vec_len (my_pending_event_vector) = 0; + smm->pending_event_vector[my_thread_index] = my_pending_event_vector; skip_dequeue: n_events = vec_len (my_fifo_events); @@ -483,8 +546,10 @@ skip_dequeue: &n_tx_packets); /* Out of buffers */ if (rv < 0) - goto done; - + { + vec_add1 (smm->pending_event_vector[my_thread_index], *e0); + continue; + } break; case FIFO_EVENT_DISCONNECT: s0 = stream_session_get_from_handle (e0->session_handle); @@ -507,25 +572,8 @@ skip_dequeue: } } -done: - - /* Couldn't process all events. Probably out of buffers */ - if (PREDICT_FALSE (i < n_events)) - { - session_fifo_event_t *partially_read = - smm->evts_partially_read[my_thread_index]; - vec_add (partially_read, &my_fifo_events[i], n_events - i); - vec_free (my_fifo_events); - smm->fifo_events[my_thread_index] = partially_read; - smm->evts_partially_read[my_thread_index] = 0; - } - else - { - vec_free (smm->fifo_events[my_thread_index]); - smm->fifo_events[my_thread_index] = - smm->evts_partially_read[my_thread_index]; - smm->evts_partially_read[my_thread_index] = 0; - } + _vec_len (my_fifo_events) = 0; + smm->free_event_vector[my_thread_index] = my_fifo_events; vlib_node_increment_counter (vm, session_queue_node.index, SESSION_QUEUE_ERROR_TX, n_tx_packets); diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c index d0792fae..c5aaf2e2 100644 --- a/src/vnet/session/session.c +++ b/src/vnet/session/session.c @@ -732,10 +732,6 @@ stream_session_connect_notify (transport_connection_t * tc, u8 sst, /* Cleanup session lookup */ stream_session_half_open_table_del (smm, sst, tc); - - /* Add to established lookup table */ - handle = (((u64) tc->thread_index) << 32) | (u64) new_s->session_index; - stream_session_table_add_for_tc (tc, handle); } void @@ -1091,11 +1087,19 @@ session_manager_main_enable (vlib_main_t * vm) vec_validate (smm->sessions, num_threads - 1); vec_validate (smm->session_indices_to_enqueue_by_thread, num_threads - 1); vec_validate (smm->tx_buffers, num_threads - 1); - vec_validate (smm->fifo_events, num_threads - 1); - vec_validate (smm->evts_partially_read, num_threads - 1); + vec_validate (smm->pending_event_vector, num_threads - 1); + vec_validate (smm->free_event_vector, num_threads - 1); vec_validate (smm->current_enqueue_epoch, num_threads - 1); vec_validate (smm->vpp_event_queues, num_threads - 1); + for (i = 0; i < num_threads; i++) + { + vec_validate (smm->free_event_vector[i], 0); + _vec_len (smm->free_event_vector[i]) = 0; + vec_validate (smm->pending_event_vector[i], 0); + _vec_len (smm->pending_event_vector[i]) = 0; + } + #if SESSION_DBG vec_validate (smm->last_event_poll_by_thread, num_threads - 1); #endif diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index a08fa231..d60cca29 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -197,10 +197,10 @@ struct _session_manager_main u32 **tx_buffers; /** Per worker-thread vector of partially read events */ - session_fifo_event_t **evts_partially_read; + session_fifo_event_t **free_event_vector; /** per-worker active event vectors */ - session_fifo_event_t **fifo_events; + session_fifo_event_t **pending_event_vector; /** vpp fifo event queue */ unix_shared_memory_queue_t **vpp_event_queues; diff --git a/src/vnet/tcp/builtin_client.c b/src/vnet/tcp/builtin_client.c index a0e61f42..d13fd446 100644 --- a/src/vnet/tcp/builtin_client.c +++ b/src/vnet/tcp/builtin_client.c @@ -44,7 +44,7 @@ #undef vl_printfun #define TCP_BUILTIN_CLIENT_DBG (1) -#define TCP_BUILTIN_CLIENT_VPP_THREAD (1) +#define TCP_BUILTIN_CLIENT_VPP_THREAD (0) #define TCP_BUILTIN_CLIENT_PTHREAD (!TCP_BUILTIN_CLIENT_VPP_THREAD) static void diff --git a/src/vnet/tcp/builtin_server.c b/src/vnet/tcp/builtin_server.c index 621ce02a..64fc4a71 100644 --- a/src/vnet/tcp/builtin_server.c +++ b/src/vnet/tcp/builtin_server.c @@ -141,16 +141,14 @@ builtin_server_rx_callback (stream_session_t * s) session_fifo_event_t evt; static int serial_number = 0; + tx_fifo = s->server_tx_fifo; + rx_fifo = s->server_rx_fifo; + max_dequeue = svm_fifo_max_dequeue (s->server_rx_fifo); max_enqueue = svm_fifo_max_enqueue (s->server_tx_fifo); if (PREDICT_FALSE (max_dequeue == 0)) - { - return 0; - } - - tx_fifo = s->server_tx_fifo; - rx_fifo = s->server_rx_fifo; + return 0; /* Number of bytes we're going to copy */ max_transfer = (max_dequeue < max_enqueue) ? max_dequeue : max_enqueue; @@ -175,8 +173,6 @@ builtin_server_rx_callback (stream_session_t * s) return 0; } - svm_fifo_unset_event (rx_fifo); - vec_validate (bsm->rx_buf, max_transfer - 1); _vec_len (bsm->rx_buf) = max_transfer; diff --git a/src/vnet/unix/gdb_funcs.c b/src/vnet/unix/gdb_funcs.c index cfb4b247..cca2e420 100644 --- a/src/vnet/unix/gdb_funcs.c +++ b/src/vnet/unix/gdb_funcs.c @@ -20,7 +20,7 @@ #include #include - +#include /** @@ -164,6 +164,16 @@ VLIB_CLI_COMMAND (show_gdb_funcs_command, static) = { .function = show_gdb_command_fn, }; +vnet_buffer_opaque_t *vb (void *vb_arg) +{ + vlib_buffer_t *b = (vlib_buffer_t *)vb_arg; + vnet_buffer_opaque_t *rv; + + rv = vnet_buffer (b); + + return rv; +} + /* Cafeteria plan, maybe you don't want these functions */ clib_error_t * gdb_func_init (vlib_main_t * vm) { return 0; } -- cgit 1.2.3-korg From 66b11318a1e5f24880e3ec77c95d70647732a4a8 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Mon, 31 Jul 2017 17:18:03 -0700 Subject: Fix tcp tx buffer allocation - Make tcp output buffer allocation macro an inline function - Use per ip version per thread tx frames for retransmits and timer events - Fix / parameterize tcp data structure preallocation - Add a couple of gdb-callable show commands - Fix local endpoint cleanup Change-Id: I67b47b7570aa14cb4634b6fd93c57cd2eacbfa29 Signed-off-by: Florin Coras Signed-off-by: Dave Barach --- src/plugins/dpdk/device/cli.c | 1 + src/vlib/error.c | 2 +- src/vlib/node_funcs.h | 16 ++-- src/vnet/ip/ip4.h | 2 - src/vnet/session/session.c | 82 +++++++++++++++-- src/vnet/session/session.h | 10 ++ src/vnet/session/session_cli.c | 2 +- src/vnet/session/session_lookup.c | 40 ++++++-- src/vnet/tcp/builtin_client.c | 39 ++++++-- src/vnet/tcp/tcp.c | 52 +++++++---- src/vnet/tcp/tcp.h | 12 ++- src/vnet/tcp/tcp_input.c | 2 + src/vnet/tcp/tcp_output.c | 188 +++++++++++++++++++++++++------------- src/vnet/unix/gdb_funcs.c | 45 ++++++++- src/vppinfra/pool.h | 2 +- 15 files changed, 375 insertions(+), 120 deletions(-) (limited to 'src/vnet/unix/gdb_funcs.c') diff --git a/src/plugins/dpdk/device/cli.c b/src/plugins/dpdk/device/cli.c index aeeb772d..fe1c41c2 100644 --- a/src/plugins/dpdk/device/cli.c +++ b/src/plugins/dpdk/device/cli.c @@ -357,6 +357,7 @@ show_dpdk_buffer (vlib_main_t * vm, unformat_input_t * input, "name=\"%s\" available = %7d allocated = %7d total = %7d\n", rmp->name, (u32) count, (u32) free_count, (u32) (count + free_count)); + rte_mempool_dump (stderr, rmp); } else { diff --git a/src/vlib/error.c b/src/vlib/error.c index e4ed4ee3..dec90bbe 100644 --- a/src/vlib/error.c +++ b/src/vlib/error.c @@ -280,7 +280,7 @@ show_errors (vlib_main_t * vm, } /* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cli_show_errors, static) = { +VLIB_CLI_COMMAND (vlib_cli_show_errors) = { .path = "show errors", .short_help = "Show error counts", .function = show_errors, diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index c0389b2f..c4c06454 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -410,19 +410,21 @@ vlib_frame_t *vlib_get_frame_to_node (vlib_main_t * vm, u32 to_node_index); void vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t * f); -always_inline vlib_process_t * -vlib_get_current_process (vlib_main_t * vm) -{ - vlib_node_main_t *nm = &vm->node_main; - return vec_elt (nm->processes, nm->current_process_index); -} - always_inline uword vlib_in_process_context (vlib_main_t * vm) { return vm->node_main.current_process_index != ~0; } +always_inline vlib_process_t * +vlib_get_current_process (vlib_main_t * vm) +{ + vlib_node_main_t *nm = &vm->node_main; + if (vlib_in_process_context (vm)) + return vec_elt (nm->processes, nm->current_process_index); + return 0; +} + always_inline uword vlib_current_process (vlib_main_t * vm) { diff --git a/src/vnet/ip/ip4.h b/src/vnet/ip/ip4.h index 8f9a8e27..74faa059 100644 --- a/src/vnet/ip/ip4.h +++ b/src/vnet/ip/ip4.h @@ -354,8 +354,6 @@ vlib_buffer_push_ip4 (vlib_main_t * vm, vlib_buffer_t * b, ih->checksum = 0; b->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_IS_IP4; vnet_buffer (b)->l3_hdr_offset = (u8 *) ih - b->data; - vnet_buffer (b)->l4_hdr_offset = vnet_buffer (b)->l3_hdr_offset + - sizeof (*ih); } else ih->checksum = ip4_header_checksum (ih); diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c index 004c7193..4ba15291 100644 --- a/src/vnet/session/session.c +++ b/src/vnet/session/session.c @@ -759,6 +759,7 @@ session_manager_main_enable (vlib_main_t * vm) session_manager_main_t *smm = &session_manager_main; vlib_thread_main_t *vtm = vlib_get_thread_main (); u32 num_threads; + u32 preallocated_sessions_per_worker; int i; num_threads = 1 /* main thread */ + vtm->n_threads; @@ -795,15 +796,35 @@ session_manager_main_enable (vlib_main_t * vm) for (i = 0; i < vec_len (smm->vpp_event_queues); i++) session_vpp_event_queue_allocate (smm, i); - /* $$$$ preallocate hack config parameter */ - for (i = 0; i < smm->preallocated_sessions; i++) + /* Preallocate sessions */ + if (num_threads == 1) { - stream_session_t *ss __attribute__ ((unused)); - pool_get_aligned (smm->sessions[0], ss, CLIB_CACHE_LINE_BYTES); + for (i = 0; i < smm->preallocated_sessions; i++) + { + stream_session_t *ss __attribute__ ((unused)); + pool_get_aligned (smm->sessions[0], ss, CLIB_CACHE_LINE_BYTES); + } + + for (i = 0; i < smm->preallocated_sessions; i++) + pool_put_index (smm->sessions[0], i); } + else + { + int j; + preallocated_sessions_per_worker = smm->preallocated_sessions / + (num_threads - 1); - for (i = 0; i < smm->preallocated_sessions; i++) - pool_put_index (smm->sessions[0], i); + for (j = 1; j < num_threads; j++) + { + for (i = 0; i < preallocated_sessions_per_worker; i++) + { + stream_session_t *ss __attribute__ ((unused)); + pool_get_aligned (smm->sessions[j], ss, CLIB_CACHE_LINE_BYTES); + } + for (i = 0; i < preallocated_sessions_per_worker; i++) + pool_put_index (smm->sessions[j], i); + } + } session_lookup_init (); @@ -863,6 +884,7 @@ session_config_fn (vlib_main_t * vm, unformat_input_t * input) { session_manager_main_t *smm = &session_manager_main; u32 nitems; + uword tmp; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { @@ -873,9 +895,53 @@ session_config_fn (vlib_main_t * vm, unformat_input_t * input) else clib_warning ("event queue length %d too small, ignored", nitems); } - if (unformat (input, "preallocated-sessions %d", - &smm->preallocated_sessions)) + else if (unformat (input, "preallocated-sessions %d", + &smm->preallocated_sessions)) + ; + else if (unformat (input, "v4-session-table-buckets %d", + &smm->configured_v4_session_table_buckets)) ; + else if (unformat (input, "v4-halfopen-table-buckets %d", + &smm->configured_v4_halfopen_table_buckets)) + ; + else if (unformat (input, "v6-session-table-buckets %d", + &smm->configured_v6_session_table_buckets)) + ; + else if (unformat (input, "v6-halfopen-table-buckets %d", + &smm->configured_v6_halfopen_table_buckets)) + ; + else if (unformat (input, "v4-session-table-memory %U", + unformat_memory_size, &tmp)) + { + if (tmp >= 0x100000000) + return clib_error_return (0, "memory size %llx (%lld) too large", + tmp, tmp); + smm->configured_v4_session_table_memory = tmp; + } + else if (unformat (input, "v4-halfopen-table-memory %U", + unformat_memory_size, &tmp)) + { + if (tmp >= 0x100000000) + return clib_error_return (0, "memory size %llx (%lld) too large", + tmp, tmp); + smm->configured_v4_halfopen_table_memory = tmp; + } + else if (unformat (input, "v6-session-table-memory %U", + unformat_memory_size, &tmp)) + { + if (tmp >= 0x100000000) + return clib_error_return (0, "memory size %llx (%lld) too large", + tmp, tmp); + smm->configured_v6_session_table_memory = tmp; + } + else if (unformat (input, "v6-halfopen-table-memory %U", + unformat_memory_size, &tmp)) + { + if (tmp >= 0x100000000) + return clib_error_return (0, "memory size %llx (%lld) too large", + tmp, tmp); + smm->configured_v6_halfopen_table_memory = tmp; + } else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index 180b9f8a..538433da 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -133,6 +133,16 @@ struct _session_manager_main /** vpp fifo event queue configured length */ u32 configured_event_queue_length; + /** session table size parameters */ + u32 configured_v4_session_table_buckets; + u32 configured_v4_session_table_memory; + u32 configured_v4_halfopen_table_buckets; + u32 configured_v4_halfopen_table_memory; + u32 configured_v6_session_table_buckets; + u32 configured_v6_session_table_memory; + u32 configured_v6_halfopen_table_buckets; + u32 configured_v6_halfopen_table_memory; + /** Unique segment name counter */ u32 unique_segment_name_counter; diff --git a/src/vnet/session/session_cli.c b/src/vnet/session/session_cli.c index de564ea7..9f3d217c 100755 --- a/src/vnet/session/session_cli.c +++ b/src/vnet/session/session_cli.c @@ -312,7 +312,7 @@ show_session_command_fn (vlib_main_t * vm, unformat_input_t * input, } /* *INDENT-OFF* */ -VLIB_CLI_COMMAND (show_session_command, static) = +VLIB_CLI_COMMAND (vlib_cli_show_session_command) = { .path = "show session", .short_help = "show session [verbose]", diff --git a/src/vnet/session/session_lookup.c b/src/vnet/session/session_lookup.c index 1ce22f80..41f9dbf0 100644 --- a/src/vnet/session/session_lookup.c +++ b/src/vnet/session/session_lookup.c @@ -569,23 +569,45 @@ stream_session_lookup_transport6 (ip6_address_t * lcl, ip6_address_t * rmt, return 0; } +#define foreach_hash_table_parameter \ + _(v4,session,buckets,20000) \ + _(v4,session,memory,(64<<20)) \ + _(v6,session,buckets,20000) \ + _(v6,session,memory,(64<<20)) \ + _(v4,halfopen,buckets,20000) \ + _(v4,halfopen,memory,(64<<20)) \ + _(v6,halfopen,buckets,20000) \ + _(v6,halfopen,memory,(64<<20)) + void session_lookup_init (void) { session_lookup_t *sl = &session_lookup; + +#define _(af,table,parm,value) \ + u32 configured_##af##_##table##_table_##parm = value; + foreach_hash_table_parameter; +#undef _ + +#define _(af,table,parm,value) \ + if (session_manager_main.configured_##af##_##table##_table_##parm) \ + configured_##af##_##table##_table_##parm = \ + session_manager_main.configured_##af##_##table##_table_##parm; + foreach_hash_table_parameter; +#undef _ + clib_bihash_init_16_8 (&sl->v4_session_hash, "v4 session table", - 200000 /* $$$$ config parameter nbuckets */ , - (64 << 20) /*$$$ config parameter table size */ ); + configured_v4_session_table_buckets, + configured_v4_session_table_memory); clib_bihash_init_48_8 (&sl->v6_session_hash, "v6 session table", - 200000 /* $$$$ config parameter nbuckets */ , - (64 << 20) /*$$$ config parameter table size */ ); - + configured_v6_session_table_buckets, + configured_v6_session_table_memory); clib_bihash_init_16_8 (&sl->v4_half_open_hash, "v4 half-open table", - 200000 /* $$$$ config parameter nbuckets */ , - (64 << 20) /*$$$ config parameter table size */ ); + configured_v4_halfopen_table_buckets, + configured_v4_halfopen_table_memory); clib_bihash_init_48_8 (&sl->v6_half_open_hash, "v6 half-open table", - 200000 /* $$$$ config parameter nbuckets */ , - (64 << 20) /*$$$ config parameter table size */ ); + configured_v6_halfopen_table_buckets, + configured_v6_halfopen_table_memory); } /* diff --git a/src/vnet/tcp/builtin_client.c b/src/vnet/tcp/builtin_client.c index 27e20f8e..48daffb4 100644 --- a/src/vnet/tcp/builtin_client.c +++ b/src/vnet/tcp/builtin_client.c @@ -597,8 +597,9 @@ clients_connect (vlib_main_t * vm, u8 * uri, u32 n_clients) a->mp = 0; vnet_connect_uri (a); - /* Crude pacing for call setups, 100k/sec */ - vlib_process_suspend (vm, 10e-6); + /* Crude pacing for call setups */ + if ((i % 4) == 0) + vlib_process_suspend (vm, 10e-6); } } @@ -612,8 +613,10 @@ test_tcp_clients_command_fn (vlib_main_t * vm, uword *event_data = 0, event_type; u8 *default_connect_uri = (u8 *) "tcp://6.0.1.1/1234", *uri; u64 tmp, total_bytes; - f64 cli_timeout = 20.0, delta; + f64 test_timeout = 20.0, syn_timeout = 20.0, delta; + f64 time_before_connects; u32 n_clients = 1; + int preallocate_sessions = 0; char *transfer_type; int i; @@ -640,7 +643,9 @@ test_tcp_clients_command_fn (vlib_main_t * vm, ; else if (unformat (input, "uri %s", &tm->connect_uri)) ; - else if (unformat (input, "cli-timeout %f", &cli_timeout)) + else if (unformat (input, "test-timeout %f", &test_timeout)) + ; + else if (unformat (input, "syn-timeout %f", &syn_timeout)) ; else if (unformat (input, "no-return")) tm->no_return = 1; @@ -657,6 +662,8 @@ test_tcp_clients_command_fn (vlib_main_t * vm, tm->private_segment_size = tmp; else if (unformat (input, "preallocate-fifos")) tm->prealloc_fifos = 1; + else if (unformat (input, "preallocate-sessions")) + preallocate_sessions = 1; else if (unformat (input, "client-batch %d", &tm->connections_per_batch)) ; @@ -674,6 +681,7 @@ test_tcp_clients_command_fn (vlib_main_t * vm, return clib_error_return (0, "failed init"); } + tm->ready_connections = 0; tm->expected_connections = n_clients; tm->rx_total = 0; @@ -705,11 +713,21 @@ test_tcp_clients_command_fn (vlib_main_t * vm, vlib_node_set_state (vlib_mains[i], builtin_client_node.index, VLIB_NODE_STATE_POLLING); + if (preallocate_sessions) + { + session_t *sp __attribute__ ((unused)); + for (i = 0; i < n_clients; i++) + pool_get (tm->sessions, sp); + for (i = 0; i < n_clients; i++) + pool_put_index (tm->sessions, i); + } + /* Fire off connect requests */ + time_before_connects = vlib_time_now (vm); clients_connect (vm, uri, n_clients); /* Park until the sessions come up, or ten seconds elapse... */ - vlib_process_wait_for_event_or_clock (vm, 10 /* timeout, seconds */ ); + vlib_process_wait_for_event_or_clock (vm, syn_timeout); event_type = vlib_process_get_events (vm, &event_data); switch (event_type) { @@ -719,6 +737,15 @@ test_tcp_clients_command_fn (vlib_main_t * vm, goto cleanup; case 1: + delta = vlib_time_now (vm) - time_before_connects; + + if (delta != 0.0) + { + vlib_cli_output + (vm, "%d three-way handshakes in %.2f seconds, %.2f/sec", + n_clients, delta, ((f64) n_clients) / delta); + } + tm->test_start_time = vlib_time_now (tm->vlib_main); vlib_cli_output (vm, "Test started at %.6f", tm->test_start_time); break; @@ -729,7 +756,7 @@ test_tcp_clients_command_fn (vlib_main_t * vm, } /* Now wait for the sessions to finish... */ - vlib_process_wait_for_event_or_clock (vm, cli_timeout); + vlib_process_wait_for_event_or_clock (vm, test_timeout); event_type = vlib_process_get_events (vm, &event_data); switch (event_type) { diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index 59b20747..8e2eb9f4 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -173,7 +173,7 @@ tcp_connection_cleanup (tcp_connection_t * tc) /* Cleanup local endpoint if this was an active connect */ tepi = transport_endpoint_lookup (&tm->local_endpoints_table, &tc->c_lcl_ip, - tc->c_lcl_port); + clib_net_to_host_u16 (tc->c_lcl_port)); if (tepi != TRANSPORT_ENDPOINT_INVALID_INDEX) { tep = pool_elt_at_index (tm->local_endpoints, tepi); @@ -367,25 +367,24 @@ tcp_allocate_local_port (ip46_address_t * ip) { tcp_main_t *tm = vnet_get_tcp_main (); transport_endpoint_t *tep; - u32 time_now, tei; + u32 tei; u16 min = 1024, max = 65535; /* XXX configurable ? */ - int tries; + int tries, limit; - tries = max - min; - time_now = tcp_time_now (); + limit = max - min; /* Only support active opens from thread 0 */ ASSERT (vlib_get_thread_index () == 0); /* Search for first free slot */ - for (; tries >= 0; tries--) + for (tries = 0; tries < limit; tries++) { u16 port = 0; /* Find a port in the specified range */ while (1) { - port = random_u32 (&time_now) & PORT_MASK; + port = random_u32 (&tm->port_allocator_seed) & PORT_MASK; if (PREDICT_TRUE (port >= min && port < max)) break; } @@ -1189,8 +1188,9 @@ tcp_main_enable (vlib_main_t * vm) vlib_thread_main_t *vtm = vlib_get_thread_main (); clib_error_t *error = 0; u32 num_threads; - int thread, i; + int i, thread; tcp_connection_t *tc __attribute__ ((unused)); + u32 preallocated_connections_per_thread; if ((error = vlib_call_init_function (vm, ip_main_init))) return error; @@ -1224,14 +1224,26 @@ tcp_main_enable (vlib_main_t * vm) vec_validate (tm->connections, num_threads - 1); /* - * Preallocate connections + * Preallocate connections. Assume that thread 0 won't + * use preallocated threads when running multi-core */ - for (thread = 0; thread < num_threads; thread++) + if (num_threads == 1) { - for (i = 0; i < tm->preallocated_connections; i++) + thread = 0; + preallocated_connections_per_thread = tm->preallocated_connections; + } + else + { + thread = 1; + preallocated_connections_per_thread = + tm->preallocated_connections / (num_threads - 1); + } + for (; thread < num_threads; thread++) + { + for (i = 0; i < preallocated_connections_per_thread; i++) pool_get (tm->connections[thread], tc); - for (i = 0; i < tm->preallocated_connections; i++) + for (i = 0; i < preallocated_connections_per_thread; i++) pool_put_index (tm->connections[thread], i); } @@ -1257,13 +1269,21 @@ tcp_main_enable (vlib_main_t * vm) / TCP_TSTAMP_RESOLUTION; clib_bihash_init_24_8 (&tm->local_endpoints_table, "local endpoint table", - 200000 /* $$$$ config parameter nbuckets */ , - (64 << 20) /*$$$ config parameter table size */ ); + 1000000 /* $$$$ config parameter nbuckets */ , + (512 << 20) /*$$$ config parameter table size */ ); + + /* Initialize [port-allocator] random number seed */ + tm->port_allocator_seed = (u32) clib_cpu_time_now (); + if (num_threads > 1) { clib_spinlock_init (&tm->half_open_lock); clib_spinlock_init (&tm->local_endpoints_lock); } + + vec_validate (tm->tx_frames[0], num_threads - 1); + vec_validate (tm->tx_frames[1], num_threads - 1); + return error; } @@ -1289,16 +1309,12 @@ clib_error_t * tcp_init (vlib_main_t * vm) { tcp_main_t *tm = vnet_get_tcp_main (); - - tm->vnet_main = vnet_get_main (); tm->is_enabled = 0; - return 0; } VLIB_INIT_FUNCTION (tcp_init); - static clib_error_t * tcp_config_fn (vlib_main_t * vm, unformat_input_t * input) { diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h index 4fa681f8..997df76f 100644 --- a/src/vnet/tcp/tcp.h +++ b/src/vnet/tcp/tcp.h @@ -369,6 +369,8 @@ typedef struct _tcp_main /** per-worker tx buffer free lists */ u32 **tx_buffers; + /** per-worker tx frames to 4/6 output nodes */ + vlib_frame_t **tx_frames[2]; /* Per worker-thread timer wheel for connections timers */ tw_timer_wheel_16t_2w_512sl_t *timer_wheels; @@ -400,11 +402,8 @@ typedef struct _tcp_main u32 last_v6_address_rotor; ip6_address_t *ip6_src_addresses; - /* convenience */ - vlib_main_t *vlib_main; - vnet_main_t *vnet_main; - ip4_main_t *ip4_main; - ip6_main_t *ip6_main; + /** Port allocator random number generator seed */ + u32 port_allocator_seed; } tcp_main_t; extern tcp_main_t tcp_main; @@ -493,6 +492,8 @@ void tcp_send_fin (tcp_connection_t * tc); void tcp_init_mss (tcp_connection_t * tc); void tcp_update_snd_mss (tcp_connection_t * tc); void tcp_update_rto (tcp_connection_t * tc); +void tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4); +void tcp_flush_frames_to_output (u8 thread_index); always_inline u32 tcp_end_seq (tcp_header_t * th, u32 len) @@ -614,6 +615,7 @@ tcp_update_time (f64 now, u32 thread_index) { tw_timer_expire_timers_16t_2w_512sl (&tcp_main.timer_wheels[thread_index], now); + tcp_flush_frames_to_output (thread_index); } u32 tcp_push_header (transport_connection_t * tconn, vlib_buffer_t * b); diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c index 6c59d70f..29f4f08d 100644 --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -1751,6 +1751,8 @@ tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node, errors = session_manager_flush_enqueue_events (my_thread_index); tcp_established_inc_counter (vm, is_ip4, TCP_ERROR_EVENT_FIFO_FULL, errors); + tcp_flush_frame_to_output (vm, my_thread_index, is_ip4); + return from_frame->n_vectors; } diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index ad13493a..f8fbb8a9 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -436,34 +436,41 @@ tcp_init_mss (tcp_connection_t * tc) tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP; } -#define tcp_get_free_buffer_index(tm, bidx) \ -do { \ - u32 *my_tx_buffers, n_free_buffers; \ - u32 thread_index = vlib_get_thread_index(); \ - my_tx_buffers = tm->tx_buffers[thread_index]; \ - if (PREDICT_FALSE(vec_len (my_tx_buffers) == 0)) \ - { \ - n_free_buffers = 32; /* TODO config or macro */ \ - vec_validate (my_tx_buffers, n_free_buffers - 1); \ - _vec_len(my_tx_buffers) = vlib_buffer_alloc_from_free_list ( \ - vlib_get_main(), my_tx_buffers, n_free_buffers, \ - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); \ - tm->tx_buffers[thread_index] = my_tx_buffers; \ - } \ - /* buffer shortage */ \ - if (PREDICT_FALSE (vec_len (my_tx_buffers) == 0)) \ - return; \ - *bidx = my_tx_buffers[_vec_len (my_tx_buffers)-1]; \ - _vec_len (my_tx_buffers) -= 1; \ -} while (0) - -#define tcp_return_buffer(tm) \ -do { \ - u32 *my_tx_buffers; \ - u32 thread_index = vlib_get_thread_index(); \ - my_tx_buffers = tm->tx_buffers[thread_index]; \ - _vec_len (my_tx_buffers) +=1; \ -} while (0) +always_inline int +tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx) +{ + u32 *my_tx_buffers, n_free_buffers; + u32 thread_index = vlib_get_thread_index (); + my_tx_buffers = tm->tx_buffers[thread_index]; + if (PREDICT_FALSE (vec_len (my_tx_buffers) == 0)) + { + n_free_buffers = VLIB_FRAME_SIZE; + vec_validate (my_tx_buffers, n_free_buffers - 1); + _vec_len (my_tx_buffers) = + vlib_buffer_alloc_from_free_list (vlib_get_main (), my_tx_buffers, + n_free_buffers, + VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); + /* buffer shortage, report failure */ + if (vec_len (my_tx_buffers) == 0) + { + clib_warning ("out of buffers"); + return -1; + } + tm->tx_buffers[thread_index] = my_tx_buffers; + } + *bidx = my_tx_buffers[_vec_len (my_tx_buffers) - 1]; + _vec_len (my_tx_buffers) -= 1; + return 0; +} + +always_inline void +tcp_return_buffer (tcp_main_t * tm) +{ + u32 *my_tx_buffers; + u32 thread_index = vlib_get_thread_index (); + my_tx_buffers = tm->tx_buffers[thread_index]; + _vec_len (my_tx_buffers) += 1; +} always_inline void tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b) @@ -706,7 +713,9 @@ tcp_send_reset (tcp_connection_t * tc, vlib_buffer_t * pkt, u8 is_ip4) ip4_header_t *ih4, *pkt_ih4; ip6_header_t *ih6, *pkt_ih6; - tcp_get_free_buffer_index (tm, &bi); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return; + b = vlib_get_buffer (vm, bi); /* Leave enough space for headers */ @@ -811,7 +820,9 @@ tcp_send_syn (tcp_connection_t * tc) u16 initial_wnd; tcp_options_t snd_opts; - tcp_get_free_buffer_index (tm, &bi); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return; + b = vlib_get_buffer (vm, bi); /* Leave enough space for headers */ @@ -854,8 +865,11 @@ tcp_send_syn (tcp_connection_t * tc) } always_inline void -tcp_enqueue_to_output (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, u8 is_ip4) +tcp_enqueue_to_output_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, + u8 is_ip4, u8 flush) { + tcp_main_t *tm = vnet_get_tcp_main (); + u32 thread_index = vlib_get_thread_index (); u32 *to_next, next_index; vlib_frame_t *f; @@ -872,12 +886,62 @@ tcp_enqueue_to_output (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, u8 is_ip4) b->pre_data[1] = next_index; } - /* Enqueue the packet */ - f = vlib_get_frame_to_node (vm, next_index); + /* Get frame to v4/6 output node */ + f = tm->tx_frames[!is_ip4][thread_index]; + if (!f) + { + f = vlib_get_frame_to_node (vm, next_index); + ASSERT (f); + tm->tx_frames[!is_ip4][thread_index] = f; + } to_next = vlib_frame_vector_args (f); - to_next[0] = bi; - f->n_vectors = 1; - vlib_put_frame_to_node (vm, next_index, f); + to_next[f->n_vectors] = bi; + f->n_vectors += 1; + if (flush || f->n_vectors == VLIB_FRAME_SIZE) + { + vlib_put_frame_to_node (vm, next_index, f); + tm->tx_frames[!is_ip4][thread_index] = 0; + } +} + +always_inline void +tcp_enqueue_to_output (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, u8 is_ip4) +{ + tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 0); +} + +always_inline void +tcp_enqueue_to_output_now (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, + u8 is_ip4) +{ + tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 1); +} + +/** + * Flush tx frame populated by retransmits and timer pops + */ +void +tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4) +{ + if (tcp_main.tx_frames[!is_ip4][thread_index]) + { + u32 next_index; + next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index; + vlib_put_frame_to_node (vm, next_index, + tcp_main.tx_frames[!is_ip4][thread_index]); + tcp_main.tx_frames[!is_ip4][thread_index] = 0; + } +} + +/** + * Flush both v4 and v6 tx frames for thread index + */ +void +tcp_flush_frames_to_output (u8 thread_index) +{ + vlib_main_t *vm = vlib_get_main (); + tcp_flush_frame_to_output (vm, thread_index, 1); + tcp_flush_frame_to_output (vm, thread_index, 0); } /** @@ -891,14 +955,15 @@ tcp_send_fin (tcp_connection_t * tc) tcp_main_t *tm = vnet_get_tcp_main (); vlib_main_t *vm = vlib_get_main (); - tcp_get_free_buffer_index (tm, &bi); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return; b = vlib_get_buffer (vm, bi); /* Leave enough space for headers */ vlib_buffer_make_headroom (b, MAX_HDRS_LEN); tcp_make_fin (tc, b); - tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4); + tcp_enqueue_to_output_now (vm, b, bi, tc->c_is_ip4); tc->flags |= TCP_CONN_FINSNT; tcp_retransmit_timer_force_update (tc); TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc); @@ -981,7 +1046,8 @@ tcp_send_ack (tcp_connection_t * tc) u32 bi; /* Get buffer */ - tcp_get_free_buffer_index (tm, &bi); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return; b = vlib_get_buffer (vm, bi); /* Fill in the ACK */ @@ -1108,7 +1174,9 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) /* Go back to first un-acked byte */ tc->snd_nxt = tc->snd_una; - tcp_get_free_buffer_index (tm, &bi); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return; + b = vlib_get_buffer (vm, bi); if (tc->state >= TCP_STATE_ESTABLISHED) @@ -1116,6 +1184,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) /* Lost FIN, retransmit and return */ if (tc->flags & TCP_CONN_FINSNT) { + tcp_return_buffer (tm); tcp_send_fin (tc); return; } @@ -1143,6 +1212,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) tcp_retransmit_timer_set (tc); ASSERT (0 || (tc->rto_boff > 1 && tc->snd_una == tc->snd_congestion)); + tcp_return_buffer (tm); return; } @@ -1164,6 +1234,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) clib_warning ("could not remove half-open connection"); ASSERT (0); } + tcp_return_buffer (tm); return; } @@ -1185,6 +1256,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) { ASSERT (tc->state == TCP_STATE_CLOSED); clib_warning ("connection closed ..."); + tcp_return_buffer (tm); return; } @@ -1254,7 +1326,9 @@ tcp_timer_persist_handler (u32 index) tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX); /* Try to force the first unsent segment */ - tcp_get_free_buffer_index (tm, &bi); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return; + b = vlib_get_buffer (vm, bi); tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una); @@ -1300,7 +1374,9 @@ tcp_retransmit_first_unacked (tcp_connection_t * tc) tc->snd_nxt = tc->snd_una; /* Get buffer */ - tcp_get_free_buffer_index (tm, &bi); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return; + b = vlib_get_buffer (vm, bi); TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2); @@ -1344,9 +1420,10 @@ tcp_fast_retransmit_sack (tcp_connection_t * tc) hole = scoreboard_get_hole (sb, sb->cur_rxt_hole); while (hole && snd_space > 0) { - tcp_get_free_buffer_index (tm, &bi); - b = vlib_get_buffer (vm, bi); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return; + b = vlib_get_buffer (vm, bi); hole = scoreboard_next_rxt_hole (sb, hole, tcp_fastrecovery_sent_1_smss (tc), &can_rescue, &snd_limited); @@ -1414,9 +1491,9 @@ tcp_fast_retransmit_no_sack (tcp_connection_t * tc) while (snd_space > 0) { - tcp_get_free_buffer_index (tm, &bi); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return; b = vlib_get_buffer (vm, bi); - offset += n_written; n_written = tcp_prepare_retransmit_segment (tc, b, offset, snd_space); @@ -1506,32 +1583,21 @@ tcp46_output_inline (vlib_main_t * vm, if (is_ip4) { - ip4_header_t *ih0; - ih0 = vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, - &tc0->c_rmt_ip4, IP_PROTOCOL_TCP, - 1); - b0->flags |= - VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; - vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data; + vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4, + IP_PROTOCOL_TCP, 1); + b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data; th0->checksum = 0; } else { ip6_header_t *ih0; - int bogus = ~0; - ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6, &tc0->c_rmt_ip6, IP_PROTOCOL_TCP); - - b0->flags |= VNET_BUFFER_F_IS_IP6 | - VNET_BUFFER_F_OFFLOAD_IP_CKSUM | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data; vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data; th0->checksum = 0; - ASSERT (!bogus); } /* Filter out DUPACKs if there are no OOO segments left */ diff --git a/src/vnet/unix/gdb_funcs.c b/src/vnet/unix/gdb_funcs.c index cca2e420..32e22d92 100644 --- a/src/vnet/unix/gdb_funcs.c +++ b/src/vnet/unix/gdb_funcs.c @@ -21,7 +21,7 @@ #include #include - +#include /** * @brief GDB callable function: vl - Return vector length of vector @@ -135,6 +135,47 @@ void vlib_runtime_index_to_node_name (u32 index) fformat(stderr, "node runtime index %d name %s\n", index, nm->nodes[index]->name); } +void gdb_show_errors (int verbose) +{ + extern vlib_cli_command_t vlib_cli_show_errors; + unformat_input_t input; + vlib_main_t * vm = vlib_get_main(); + + if (verbose == 0) + unformat_init_string (&input, "verbose 0", 9); + else if (verbose == 1) + unformat_init_string (&input, "verbose 1", 9); + else + { + fformat(stderr, "verbose not 0 or 1\n"); + return; + } + + vlib_cli_show_errors.function (vm, &input, 0 /* cmd */); + unformat_free (&input); +} + +void gdb_show_session (int verbose) +{ + extern vlib_cli_command_t vlib_cli_show_session_command; + unformat_input_t input; + vlib_main_t * vm = vlib_get_main(); + + if (verbose == 0) + unformat_init_string (&input, "verbose 0", 9); + else if (verbose == 1) + unformat_init_string (&input, "verbose 1", 9); + else if (verbose == 2) + unformat_init_string (&input, "verbose 2", 9); + else + { + fformat(stderr, "verbose not 0 - 2\n"); + return; + } + + vlib_cli_show_session_command.function (vm, &input, 0 /* cmd */); + unformat_free (&input); +} /** * @brief GDB callable function: show_gdb_command_fn - show gdb @@ -151,6 +192,8 @@ show_gdb_command_fn (vlib_main_t * vm, vlib_cli_output (vm, "vl(p) returns vec_len(p)"); vlib_cli_output (vm, "pe(p) returns pool_elts(p)"); vlib_cli_output (vm, "pifi(p, i) returns pool_is_free_index(p, i)"); + vlib_cli_output (vm, "gdb_show_errors(0|1) dumps error counters"); + vlib_cli_output (vm, "gdb_show_session dumps session counters"); vlib_cli_output (vm, "debug_hex_bytes (ptr, n_bytes) dumps n_bytes in hex"); vlib_cli_output (vm, "vlib_dump_frame_ownership() does what it says"); vlib_cli_output (vm, "vlib_runtime_index_to_node_name (index) prints NN"); diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h index 57838e1c..56536b77 100644 --- a/src/vppinfra/pool.h +++ b/src/vppinfra/pool.h @@ -200,7 +200,7 @@ do { \ #define pool_get(P,E) pool_get_aligned(P,E,0) /** See if pool_get will expand the pool or not */ -#define pool_get_aligned_will_expand (P,YESNO,A) \ +#define pool_get_aligned_will_expand(P,YESNO,A) \ do { \ pool_header_t * _pool_var (p) = pool_header (P); \ uword _pool_var (l); \ -- cgit 1.2.3-korg