From a7575eacb922965e642e38560be3a2f34c5a6eff Mon Sep 17 00:00:00 2001 From: Mohsin KAZMI Date: Thu, 16 Jun 2016 06:58:34 +0200 Subject: netmap: multithreading support This patch adds multithreading support for netmap interfaces. Change-Id: Iba94386fe309a4aac71646fe567f8dabbebd0459 Signed-off-by: Mohsin KAZMI --- vnet/vnet/devices/netmap/device.c | 9 +++++++++ vnet/vnet/devices/netmap/netmap.c | 34 ++++++++++++++++++++++++++++++++++ vnet/vnet/devices/netmap/netmap.h | 9 ++++++++- vnet/vnet/devices/netmap/node.c | 34 +++++++++++++++++++--------------- 4 files changed, 70 insertions(+), 16 deletions(-) (limited to 'vnet') diff --git a/vnet/vnet/devices/netmap/device.c b/vnet/vnet/devices/netmap/device.c index a966ffef..a1332603 100644 --- a/vnet/vnet/devices/netmap/device.c +++ b/vnet/vnet/devices/netmap/device.c @@ -103,6 +103,12 @@ netmap_interface_tx (vlib_main_t * vm, netmap_if_t * nif = pool_elt_at_index (nm->interfaces, rd->dev_instance); int cur_ring; + if (PREDICT_FALSE(nif->lockp != 0)) + { + while (__sync_lock_test_and_set (nif->lockp, 1)) + ; + } + cur_ring = nif->first_tx_ring; while(n_left && cur_ring <= nif->last_tx_ring) @@ -156,6 +162,9 @@ netmap_interface_tx (vlib_main_t * vm, if (n_left < frame->n_vectors) ioctl(nif->fd, NIOCTXSYNC, NULL); + if (PREDICT_FALSE(nif->lockp != 0)) + *nif->lockp = 0; + if (n_left) vlib_error_count (vm, node->node_index, (n_left == frame->n_vectors ? NETMAP_TX_ERROR_PENDING_MSGS : NETMAP_TX_ERROR_NO_FREE_SLOTS), n_left); diff --git a/vnet/vnet/devices/netmap/netmap.c b/vnet/vnet/devices/netmap/netmap.c index df3b2be4..ed90862f 100644 --- a/vnet/vnet/devices/netmap/netmap.c +++ b/vnet/vnet/devices/netmap/netmap.c @@ -92,6 +92,7 @@ netmap_create_if(vlib_main_t * vm, u8 * if_name, u8 * hw_addr_set, uword * p; struct nmreq * req = 0; netmap_mem_region_t * reg; + vlib_thread_main_t * tm = vlib_get_thread_main(); int fd; p = mhash_get (&nm->if_index_by_host_if_name, if_name); @@ -152,6 +153,13 @@ netmap_create_if(vlib_main_t * vm, u8 * if_name, u8 * hw_addr_set, nif->host_if_name = if_name; nif->per_interface_next_index = ~0; + if (tm->n_vlib_mains > 1) + { + nif->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, + CLIB_CACHE_LINE_BYTES); + memset ((void *) nif->lockp, 0, CLIB_CACHE_LINE_BYTES); + } + { unix_file_t template = {0}; template.read_function = netmap_fd_read_ready; @@ -232,11 +240,37 @@ static clib_error_t * netmap_init (vlib_main_t * vm) { netmap_main_t * nm = &netmap_main; + vlib_thread_main_t * tm = vlib_get_thread_main(); + vlib_thread_registration_t * tr; + uword * p; memset (nm, 0, sizeof (netmap_main_t)); + nm->input_cpu_first_index = 0; + nm->input_cpu_count = 1; + + /* find out which cpus will be used for input */ + p = hash_get_mem (tm->thread_registrations_by_name, "workers"); + tr = p ? (vlib_thread_registration_t *) p[0] : 0; + + if (tr && tr->count > 0) + { + nm->input_cpu_first_index = tr->first_index; + nm->input_cpu_count = tr->count; + } + + /* if worker threads are enabled, switch to polling mode */ + if (tm->n_vlib_mains > 1) + foreach_vlib_main ( + ({ + vlib_node_set_state(this_vlib_main, netmap_input_node.index, VLIB_NODE_STATE_POLLING); + })); + mhash_init_vec_string (&nm->if_index_by_host_if_name, sizeof (uword)); + vec_validate_aligned (nm->rx_buffers, tm->n_vlib_mains - 1, + CLIB_CACHE_LINE_BYTES); + return 0; } diff --git a/vnet/vnet/devices/netmap/netmap.h b/vnet/vnet/devices/netmap/netmap.h index 6f7791c3..d29057fe 100644 --- a/vnet/vnet/devices/netmap/netmap.h +++ b/vnet/vnet/devices/netmap/netmap.h @@ -42,6 +42,7 @@ typedef struct { CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + volatile u32 * lockp; u8 * host_if_name; uword if_index; u32 hw_if_index; @@ -77,13 +78,19 @@ typedef struct { uword * pending_input_bitmap; /* rx buffer cache */ - u32 * rx_buffers; + u32 ** rx_buffers; /* hash of host interface names */ mhash_t if_index_by_host_if_name; /* vector of memory regions */ netmap_mem_region_t * mem_regions; + + /* first cpu index */ + u32 input_cpu_first_index; + + /* total cpu count */ + u32 input_cpu_count; } netmap_main_t; netmap_main_t netmap_main; diff --git a/vnet/vnet/devices/netmap/node.c b/vnet/vnet/devices/netmap/node.c index eae189c0..9378dfd9 100644 --- a/vnet/vnet/devices/netmap/node.c +++ b/vnet/vnet/devices/netmap/node.c @@ -99,30 +99,30 @@ buffer_add_to_chain(vlib_main_t *vm, u32 bi, u32 first_bi, u32 prev_bi) always_inline uword netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame, u32 device_idx) + vlib_frame_t * frame, netmap_if_t * nif) { u32 next_index = NETMAP_INPUT_NEXT_ETHERNET_INPUT; uword n_trace = vlib_get_trace_count (vm, node); netmap_main_t * nm = &netmap_main; - netmap_if_t * nif = pool_elt_at_index(nm->interfaces, device_idx); u32 n_rx_packets = 0; u32 n_rx_bytes = 0; u32 * to_next = 0; u32 n_free_bufs; struct netmap_ring * ring; int cur_ring; + u32 cpu_index = os_get_cpu_number(); u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); if (nif->per_interface_next_index != ~0) next_index = nif->per_interface_next_index; - n_free_bufs = vec_len (nm->rx_buffers); + n_free_bufs = vec_len (nm->rx_buffers[cpu_index]); if (PREDICT_FALSE(n_free_bufs < VLIB_FRAME_SIZE)) { - vec_validate(nm->rx_buffers, VLIB_FRAME_SIZE + n_free_bufs - 1); - n_free_bufs += vlib_buffer_alloc(vm, &nm->rx_buffers[n_free_bufs], VLIB_FRAME_SIZE); - _vec_len (nm->rx_buffers) = n_free_bufs; + vec_validate(nm->rx_buffers[cpu_index], VLIB_FRAME_SIZE + n_free_bufs - 1); + n_free_bufs += vlib_buffer_alloc(vm, &nm->rx_buffers[cpu_index][n_free_bufs], VLIB_FRAME_SIZE); + _vec_len (nm->rx_buffers[cpu_index]) = n_free_bufs; } cur_ring = nif->first_rx_ring; @@ -168,11 +168,11 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, while (data_len && n_free_bufs) { /* grab free buffer */ - u32 last_empty_buffer = vec_len (nm->rx_buffers) - 1; + u32 last_empty_buffer = vec_len (nm->rx_buffers[cpu_index]) - 1; prev_bi0 = bi0; - bi0 = nm->rx_buffers[last_empty_buffer]; + bi0 = nm->rx_buffers[cpu_index][last_empty_buffer]; b0 = vlib_get_buffer (vm, bi0); - _vec_len (nm->rx_buffers) = last_empty_buffer; + _vec_len (nm->rx_buffers[cpu_index]) = last_empty_buffer; n_free_bufs--; /* copy data */ @@ -257,14 +257,17 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, { int i; u32 n_rx_packets = 0; - + u32 cpu_index = os_get_cpu_number(); netmap_main_t * nm = &netmap_main; + netmap_if_t * nmi; - clib_bitmap_foreach (i, nm->pending_input_bitmap, - ({ - clib_bitmap_set (nm->pending_input_bitmap, i, 0); - n_rx_packets += netmap_device_input_fn(vm, node, frame, i); - })); + for(i = 0; i < vec_len(nm->interfaces); i++ ) + { + nmi = vec_elt_at_index(nm->interfaces, i); + if (nmi->is_admin_up && + (i % nm->input_cpu_count) == (cpu_index - nm->input_cpu_first_index)) + n_rx_packets += netmap_device_input_fn(vm, node, frame, nmi); + } return n_rx_packets; } @@ -274,6 +277,7 @@ VLIB_REGISTER_NODE (netmap_input_node) = { .name = "netmap-input", .format_trace = format_netmap_input_trace, .type = VLIB_NODE_TYPE_INPUT, + /* default state is INTERRUPT mode, switch to POLLING if worker threads are enabled */ .state = VLIB_NODE_STATE_INTERRUPT, .n_errors = NETMAP_INPUT_N_ERROR, .error_strings = netmap_input_error_strings, -- cgit 1.2.3-korg