aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--vnet/vnet/devices/netmap/device.c9
-rw-r--r--vnet/vnet/devices/netmap/netmap.c34
-rw-r--r--vnet/vnet/devices/netmap/netmap.h9
-rw-r--r--vnet/vnet/devices/netmap/node.c34
4 files changed, 70 insertions, 16 deletions
diff --git a/vnet/vnet/devices/netmap/device.c b/vnet/vnet/devices/netmap/device.c
index a966ffef668..a13326035de 100644
--- a/vnet/vnet/devices/netmap/device.c
+++ b/vnet/vnet/devices/netmap/device.c
@@ -103,6 +103,12 @@ netmap_interface_tx (vlib_main_t * vm,
netmap_if_t * nif = pool_elt_at_index (nm->interfaces, rd->dev_instance);
int cur_ring;
+ if (PREDICT_FALSE(nif->lockp != 0))
+ {
+ while (__sync_lock_test_and_set (nif->lockp, 1))
+ ;
+ }
+
cur_ring = nif->first_tx_ring;
while(n_left && cur_ring <= nif->last_tx_ring)
@@ -156,6 +162,9 @@ netmap_interface_tx (vlib_main_t * vm,
if (n_left < frame->n_vectors)
ioctl(nif->fd, NIOCTXSYNC, NULL);
+ if (PREDICT_FALSE(nif->lockp != 0))
+ *nif->lockp = 0;
+
if (n_left)
vlib_error_count (vm, node->node_index,
(n_left == frame->n_vectors ? NETMAP_TX_ERROR_PENDING_MSGS : NETMAP_TX_ERROR_NO_FREE_SLOTS), n_left);
diff --git a/vnet/vnet/devices/netmap/netmap.c b/vnet/vnet/devices/netmap/netmap.c
index df3b2be499e..ed90862fdf0 100644
--- a/vnet/vnet/devices/netmap/netmap.c
+++ b/vnet/vnet/devices/netmap/netmap.c
@@ -92,6 +92,7 @@ netmap_create_if(vlib_main_t * vm, u8 * if_name, u8 * hw_addr_set,
uword * p;
struct nmreq * req = 0;
netmap_mem_region_t * reg;
+ vlib_thread_main_t * tm = vlib_get_thread_main();
int fd;
p = mhash_get (&nm->if_index_by_host_if_name, if_name);
@@ -152,6 +153,13 @@ netmap_create_if(vlib_main_t * vm, u8 * if_name, u8 * hw_addr_set,
nif->host_if_name = if_name;
nif->per_interface_next_index = ~0;
+ if (tm->n_vlib_mains > 1)
+ {
+ nif->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
+ CLIB_CACHE_LINE_BYTES);
+ memset ((void *) nif->lockp, 0, CLIB_CACHE_LINE_BYTES);
+ }
+
{
unix_file_t template = {0};
template.read_function = netmap_fd_read_ready;
@@ -232,11 +240,37 @@ static clib_error_t *
netmap_init (vlib_main_t * vm)
{
netmap_main_t * nm = &netmap_main;
+ vlib_thread_main_t * tm = vlib_get_thread_main();
+ vlib_thread_registration_t * tr;
+ uword * p;
memset (nm, 0, sizeof (netmap_main_t));
+ nm->input_cpu_first_index = 0;
+ nm->input_cpu_count = 1;
+
+ /* find out which cpus will be used for input */
+ p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+ tr = p ? (vlib_thread_registration_t *) p[0] : 0;
+
+ if (tr && tr->count > 0)
+ {
+ nm->input_cpu_first_index = tr->first_index;
+ nm->input_cpu_count = tr->count;
+ }
+
+ /* if worker threads are enabled, switch to polling mode */
+ if (tm->n_vlib_mains > 1)
+ foreach_vlib_main (
+ ({
+ vlib_node_set_state(this_vlib_main, netmap_input_node.index, VLIB_NODE_STATE_POLLING);
+ }));
+
mhash_init_vec_string (&nm->if_index_by_host_if_name, sizeof (uword));
+ vec_validate_aligned (nm->rx_buffers, tm->n_vlib_mains - 1,
+ CLIB_CACHE_LINE_BYTES);
+
return 0;
}
diff --git a/vnet/vnet/devices/netmap/netmap.h b/vnet/vnet/devices/netmap/netmap.h
index 6f7791c3732..d29057febcf 100644
--- a/vnet/vnet/devices/netmap/netmap.h
+++ b/vnet/vnet/devices/netmap/netmap.h
@@ -42,6 +42,7 @@
typedef struct {
CLIB_CACHE_LINE_ALIGN_MARK(cacheline0);
+ volatile u32 * lockp;
u8 * host_if_name;
uword if_index;
u32 hw_if_index;
@@ -77,13 +78,19 @@ typedef struct {
uword * pending_input_bitmap;
/* rx buffer cache */
- u32 * rx_buffers;
+ u32 ** rx_buffers;
/* hash of host interface names */
mhash_t if_index_by_host_if_name;
/* vector of memory regions */
netmap_mem_region_t * mem_regions;
+
+ /* first cpu index */
+ u32 input_cpu_first_index;
+
+ /* total cpu count */
+ u32 input_cpu_count;
} netmap_main_t;
netmap_main_t netmap_main;
diff --git a/vnet/vnet/devices/netmap/node.c b/vnet/vnet/devices/netmap/node.c
index eae189c0e64..9378dfd95bb 100644
--- a/vnet/vnet/devices/netmap/node.c
+++ b/vnet/vnet/devices/netmap/node.c
@@ -99,30 +99,30 @@ buffer_add_to_chain(vlib_main_t *vm, u32 bi, u32 first_bi, u32 prev_bi)
always_inline uword
netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
- vlib_frame_t * frame, u32 device_idx)
+ vlib_frame_t * frame, netmap_if_t * nif)
{
u32 next_index = NETMAP_INPUT_NEXT_ETHERNET_INPUT;
uword n_trace = vlib_get_trace_count (vm, node);
netmap_main_t * nm = &netmap_main;
- netmap_if_t * nif = pool_elt_at_index(nm->interfaces, device_idx);
u32 n_rx_packets = 0;
u32 n_rx_bytes = 0;
u32 * to_next = 0;
u32 n_free_bufs;
struct netmap_ring * ring;
int cur_ring;
+ u32 cpu_index = os_get_cpu_number();
u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm,
VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
if (nif->per_interface_next_index != ~0)
next_index = nif->per_interface_next_index;
- n_free_bufs = vec_len (nm->rx_buffers);
+ n_free_bufs = vec_len (nm->rx_buffers[cpu_index]);
if (PREDICT_FALSE(n_free_bufs < VLIB_FRAME_SIZE))
{
- vec_validate(nm->rx_buffers, VLIB_FRAME_SIZE + n_free_bufs - 1);
- n_free_bufs += vlib_buffer_alloc(vm, &nm->rx_buffers[n_free_bufs], VLIB_FRAME_SIZE);
- _vec_len (nm->rx_buffers) = n_free_bufs;
+ vec_validate(nm->rx_buffers[cpu_index], VLIB_FRAME_SIZE + n_free_bufs - 1);
+ n_free_bufs += vlib_buffer_alloc(vm, &nm->rx_buffers[cpu_index][n_free_bufs], VLIB_FRAME_SIZE);
+ _vec_len (nm->rx_buffers[cpu_index]) = n_free_bufs;
}
cur_ring = nif->first_rx_ring;
@@ -168,11 +168,11 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
while (data_len && n_free_bufs)
{
/* grab free buffer */
- u32 last_empty_buffer = vec_len (nm->rx_buffers) - 1;
+ u32 last_empty_buffer = vec_len (nm->rx_buffers[cpu_index]) - 1;
prev_bi0 = bi0;
- bi0 = nm->rx_buffers[last_empty_buffer];
+ bi0 = nm->rx_buffers[cpu_index][last_empty_buffer];
b0 = vlib_get_buffer (vm, bi0);
- _vec_len (nm->rx_buffers) = last_empty_buffer;
+ _vec_len (nm->rx_buffers[cpu_index]) = last_empty_buffer;
n_free_bufs--;
/* copy data */
@@ -257,14 +257,17 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
{
int i;
u32 n_rx_packets = 0;
-
+ u32 cpu_index = os_get_cpu_number();
netmap_main_t * nm = &netmap_main;
+ netmap_if_t * nmi;
- clib_bitmap_foreach (i, nm->pending_input_bitmap,
- ({
- clib_bitmap_set (nm->pending_input_bitmap, i, 0);
- n_rx_packets += netmap_device_input_fn(vm, node, frame, i);
- }));
+ for(i = 0; i < vec_len(nm->interfaces); i++ )
+ {
+ nmi = vec_elt_at_index(nm->interfaces, i);
+ if (nmi->is_admin_up &&
+ (i % nm->input_cpu_count) == (cpu_index - nm->input_cpu_first_index))
+ n_rx_packets += netmap_device_input_fn(vm, node, frame, nmi);
+ }
return n_rx_packets;
}
@@ -274,6 +277,7 @@ VLIB_REGISTER_NODE (netmap_input_node) = {
.name = "netmap-input",
.format_trace = format_netmap_input_trace,
.type = VLIB_NODE_TYPE_INPUT,
+ /* default state is INTERRUPT mode, switch to POLLING if worker threads are enabled */
.state = VLIB_NODE_STATE_INTERRUPT,
.n_errors = NETMAP_INPUT_N_ERROR,
.error_strings = netmap_input_error_strings,