summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--vlib/vlib/threads.h23
-rw-r--r--vnet/vnet/ip/icmp4.c2
-rw-r--r--vnet/vnet/ip/icmp6.c2
-rw-r--r--vnet/vnet/ip/ip4_pg.c2
-rw-r--r--vnet/vnet/ip/ip6_pg.c2
-rw-r--r--vnet/vnet/ip/udp_pg.c2
-rw-r--r--vnet/vnet/pg/cli.c8
-rw-r--r--vnet/vnet/pg/edit.c3
-rw-r--r--vnet/vnet/pg/init.c1
-rw-r--r--vnet/vnet/pg/input.c31
-rw-r--r--vnet/vnet/pg/output.c6
-rw-r--r--vnet/vnet/pg/pg.h9
-rw-r--r--vnet/vnet/pg/stream.c45
13 files changed, 98 insertions, 38 deletions
diff --git a/vlib/vlib/threads.h b/vlib/vlib/threads.h
index 2772fa86e0f..e30b9e35118 100644
--- a/vlib/vlib/threads.h
+++ b/vlib/vlib/threads.h
@@ -353,6 +353,29 @@ static void __vlib_add_thread_registration_##x (void) \
} \
__VA_ARGS__ vlib_thread_registration_t x
+always_inline u32
+vlib_num_workers ()
+{
+ return vlib_thread_main.n_vlib_mains - 1;
+}
+
+always_inline u32
+vlib_get_worker_cpu_index (u32 worker_index)
+{
+ return worker_index + 1;
+}
+
+always_inline vlib_main_t *
+vlib_get_worker_vlib_main (u32 worker_index)
+{
+ vlib_main_t *vm;
+ vlib_thread_main_t *tm = &vlib_thread_main;
+ ASSERT (worker_index < tm->n_vlib_mains - 1);
+ vm = vlib_mains[worker_index + 1];
+ ASSERT (vm);
+ return vm;
+}
+
#endif /* included_vlib_threads_h */
/*
diff --git a/vnet/vnet/ip/icmp4.c b/vnet/vnet/ip/icmp4.c
index a02b52950a3..c160f88a600 100644
--- a/vnet/vnet/ip/icmp4.c
+++ b/vnet/vnet/ip/icmp4.c
@@ -601,7 +601,7 @@ icmp4_pg_edit_function (pg_main_t * pg,
u32 * packets,
u32 n_packets)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_offset, icmp_offset;
icmp_offset = g->start_byte_offset;
diff --git a/vnet/vnet/ip/icmp6.c b/vnet/vnet/ip/icmp6.c
index 313ab98d169..a04373a3a7e 100644
--- a/vnet/vnet/ip/icmp6.c
+++ b/vnet/vnet/ip/icmp6.c
@@ -678,7 +678,7 @@ icmp6_pg_edit_function (pg_main_t * pg,
u32 * packets,
u32 n_packets)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_offset, icmp_offset;
int bogus_length;
diff --git a/vnet/vnet/ip/ip4_pg.c b/vnet/vnet/ip/ip4_pg.c
index 9710d8d4c5a..42dad85272a 100644
--- a/vnet/vnet/ip/ip4_pg.c
+++ b/vnet/vnet/ip/ip4_pg.c
@@ -130,7 +130,7 @@ ip4_pg_edit_function (pg_main_t * pg,
u32 * packets,
u32 n_packets)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_offset;
ip_offset = g->start_byte_offset;
diff --git a/vnet/vnet/ip/ip6_pg.c b/vnet/vnet/ip/ip6_pg.c
index 2c3852765d4..872f197076a 100644
--- a/vnet/vnet/ip/ip6_pg.c
+++ b/vnet/vnet/ip/ip6_pg.c
@@ -47,7 +47,7 @@ ip6_pg_edit_function (pg_main_t * pg,
u32 * packets,
u32 n_packets)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_header_offset = g->start_byte_offset;
while (n_packets >= 2)
diff --git a/vnet/vnet/ip/udp_pg.c b/vnet/vnet/ip/udp_pg.c
index a33a56294fb..76383a9a9c2 100644
--- a/vnet/vnet/ip/udp_pg.c
+++ b/vnet/vnet/ip/udp_pg.c
@@ -51,7 +51,7 @@ udp_pg_edit_function_inline (pg_main_t * pg,
u32 n_packets,
u32 flags)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_offset, udp_offset;
udp_offset = g->start_byte_offset;
diff --git a/vnet/vnet/pg/cli.c b/vnet/vnet/pg/cli.c
index e6bd11e09ce..f5896b4326e 100644
--- a/vnet/vnet/pg/cli.c
+++ b/vnet/vnet/pg/cli.c
@@ -173,6 +173,8 @@ format_pg_stream (u8 * s, va_list * va)
v = format (v, "buffer-size %d, ", t->buffer_bytes);
+ v = format (v, "worker %d, ", t->worker_index);
+
if (v)
{
s = format (s, " %v", v);
@@ -330,6 +332,9 @@ new_stream (vlib_main_t * vm,
unformat_vlib_node, vm, &s.node_index))
;
+ else if (unformat (input, "worker %u", &s.worker_index))
+ ;
+
else if (unformat (input, "interface %U",
unformat_vnet_sw_interface, vnm,
&s.sw_if_index[VLIB_RX]))
@@ -389,6 +394,9 @@ new_stream (vlib_main_t * vm,
else
n = 0;
+ if (s.worker_index >= vlib_num_workers ())
+ s.worker_index = 0;
+
if (pcap_file_name != 0)
{
error = pg_pcap_read (&s, pcap_file_name);
diff --git a/vnet/vnet/pg/edit.c b/vnet/vnet/pg/edit.c
index c6fc3fd124e..cb4d070fb19 100644
--- a/vnet/vnet/pg/edit.c
+++ b/vnet/vnet/pg/edit.c
@@ -127,8 +127,7 @@ uword
unformat_pg_payload (unformat_input_t * input, va_list * args)
{
pg_stream_t *s = va_arg (*args, pg_stream_t *);
- pg_main_t *pg = &pg_main;
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
pg_edit_t *e;
u32 i, node_index, len, max_len;
u8 *v;
diff --git a/vnet/vnet/pg/init.c b/vnet/vnet/pg/init.c
index d939a31e1d0..631be25ea3c 100644
--- a/vnet/vnet/pg/init.c
+++ b/vnet/vnet/pg/init.c
@@ -49,7 +49,6 @@ pg_init (vlib_main_t * vm)
clib_error_t *error;
pg_main_t *pg = &pg_main;
- pg->vlib_main = vm;
pg->if_index_by_if_id = hash_create (0, sizeof (uword));
if ((error = vlib_call_init_function (vm, vnet_main_init)))
diff --git a/vnet/vnet/pg/input.c b/vnet/vnet/pg/input.c
index af81d5386cb..46c8f4fa2f7 100644
--- a/vnet/vnet/pg/input.c
+++ b/vnet/vnet/pg/input.c
@@ -194,7 +194,7 @@ do_set_fixed (pg_main_t * pg,
u32 n_bits,
u32 byte_offset, u32 is_net_byte_order, u64 v_min, u64 v_max)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
while (n_buffers >= 4)
{
@@ -247,7 +247,7 @@ do_set_increment (pg_main_t * pg,
u32 is_net_byte_order,
u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max, u64 v)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u64 sum = 0;
ASSERT (v >= v_min && v <= v_max);
@@ -342,7 +342,7 @@ do_set_random (pg_main_t * pg,
u32 is_net_byte_order,
u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u64 v_diff = v_max - v_min + 1;
u64 r_mask = max_pow2 (v_diff) - 1;
u64 v0, v1;
@@ -533,7 +533,7 @@ do_setbits_fixed (pg_main_t * pg,
u32 n_bits,
u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
while (n_buffers >= 4)
{
@@ -586,7 +586,7 @@ do_setbits_increment (pg_main_t * pg,
u32 byte_offset,
u64 v_min, u64 v_max, u64 v, u64 mask, u32 shift)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
ASSERT (v >= v_min && v <= v_max);
@@ -664,7 +664,7 @@ do_setbits_random (pg_main_t * pg,
u32 n_bits,
u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u64 v_diff = v_max - v_min + 1;
u64 r_mask = max_pow2 (v_diff) - 1;
u64 v0, v1;
@@ -938,7 +938,7 @@ pg_generate_fix_multi_buffer_lengths (pg_main_t * pg,
pg_stream_t * s,
u32 * buffers, u32 n_buffers)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
pg_buffer_index_t *pbi;
uword n_bytes_left;
static u32 *unused_buffers = 0;
@@ -1046,7 +1046,7 @@ pg_set_next_buffer_pointers (pg_main_t * pg,
pg_stream_t * s,
u32 * buffers, u32 * next_buffers, u32 n_buffers)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
while (n_buffers >= 4)
{
@@ -1237,7 +1237,7 @@ pg_stream_fill_helper (pg_main_t * pg,
pg_buffer_index_t * bi,
u32 * buffers, u32 * next_buffers, u32 n_alloc)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
vlib_buffer_free_list_t *f;
uword is_start_of_packet = bi == s->buffer_indices;
u32 n_allocated;
@@ -1477,7 +1477,7 @@ pg_input_trace (pg_main_t * pg,
vlib_node_runtime_t * node,
pg_stream_t * s, u32 * buffers, u32 n_buffers)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u32 *b, n_left, stream_index, next_index;
n_left = n_buffers;
@@ -1548,7 +1548,7 @@ pg_generate_packets (vlib_node_runtime_t * node,
pg_main_t * pg,
pg_stream_t * s, uword n_packets_to_generate)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u32 *to_next, n_this_frame, n_left, n_trace, n_packets_in_fifo;
uword n_packets_generated;
pg_buffer_index_t *bi, *bi0;
@@ -1632,7 +1632,7 @@ pg_generate_packets (vlib_node_runtime_t * node,
static uword
pg_input_stream (vlib_node_runtime_t * node, pg_main_t * pg, pg_stream_t * s)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
uword n_packets;
f64 time_now, dt;
@@ -1683,10 +1683,15 @@ pg_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
uword i;
pg_main_t *pg = &pg_main;
uword n_packets = 0;
+ u32 num_workers = vlib_num_workers ();
+ u32 cpu_index = os_get_cpu_number ();
/* *INDENT-OFF* */
clib_bitmap_foreach (i, pg->enabled_streams, ({
- n_packets += pg_input_stream (node, pg, vec_elt_at_index (pg->streams, i));
+ pg_stream_t *s = vec_elt_at_index (pg->streams, i);
+ if (num_workers == 0 ||
+ vlib_get_worker_cpu_index (s->worker_index) == cpu_index)
+ n_packets += pg_input_stream (node, pg, s);
}));
/* *INDENT-ON* */
diff --git a/vnet/vnet/pg/output.c b/vnet/vnet/pg/output.c
index 2de06acaea5..64a0b7b9d92 100644
--- a/vnet/vnet/pg/output.c
+++ b/vnet/vnet/pg/output.c
@@ -52,6 +52,10 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
pg_interface_t *pif = pool_elt_at_index (pg->interfaces, rd->dev_instance);
+ if (PREDICT_FALSE (pif->lockp != 0))
+ while (__sync_lock_test_and_set (pif->lockp, 1))
+ ;
+
if (pif->pcap_file_name != 0)
{
while (n_left > 0)
@@ -67,6 +71,8 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
}
vlib_buffer_free_no_next (vm, vlib_frame_args (frame), n_buffers);
+ if (PREDICT_FALSE (pif->lockp != 0))
+ *pif->lockp = 0;
return n_buffers;
}
diff --git a/vnet/vnet/pg/pg.h b/vnet/vnet/pg/pg.h
index 067172da174..1cd42229882 100644
--- a/vnet/vnet/pg/pg.h
+++ b/vnet/vnet/pg/pg.h
@@ -143,6 +143,9 @@ typedef struct pg_stream_t
/* Node where stream's buffers get put. */
u32 node_index;
+ /* Worker thread index */
+ u32 worker_index;
+
/* Output next index to reach output node from stream input node. */
u32 next_index;
@@ -284,6 +287,9 @@ pg_free_edit_group (pg_stream_t * s)
typedef struct
{
+ /* TX lock */
+ volatile u32 *lockp;
+
/* VLIB interface indices. */
u32 hw_if_index, sw_if_index;
@@ -303,9 +309,6 @@ typedef struct
typedef struct pg_main_t
{
- /* Back pointer to main structure. */
- vlib_main_t *vlib_main;
-
/* Pool of streams. */
pg_stream_t *streams;
diff --git a/vnet/vnet/pg/stream.c b/vnet/vnet/pg/stream.c
index 20f54a74b8c..eba99419b5a 100644
--- a/vnet/vnet/pg/stream.c
+++ b/vnet/vnet/pg/stream.c
@@ -47,6 +47,7 @@
void
pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s, int want_enabled)
{
+ vlib_main_t *vm;
vnet_main_t *vnm = vnet_get_main ();
pg_interface_t *pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
@@ -76,11 +77,14 @@ pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s, int want_enabled)
VNET_SW_INTERFACE_FLAG_ADMIN_UP);
}
- vlib_node_set_state (pg->vlib_main,
- pg_input_node.index,
- (clib_bitmap_is_zero (pg->enabled_streams)
- ? VLIB_NODE_STATE_DISABLED
- : VLIB_NODE_STATE_POLLING));
+ if (vlib_num_workers ())
+ vm = vlib_get_worker_vlib_main (s->worker_index);
+ else
+ vm = vlib_get_main ();
+
+ vlib_node_set_state (vm, pg_input_node.index,
+ (clib_bitmap_is_zero (pg->enabled_streams) ?
+ VLIB_NODE_STATE_DISABLED : VLIB_NODE_STATE_POLLING));
s->packet_accumulator = 0;
s->time_last_generate = 0;
@@ -189,6 +193,13 @@ pg_interface_add_or_get (pg_main_t * pg, uword if_id)
hash_set (pg->if_index_by_if_id, if_id, i);
+ if (vlib_num_workers ())
+ {
+ pi->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
+ CLIB_CACHE_LINE_BYTES);
+ *pi->lockp = 0;
+ }
+
ip4_sw_interface_enable_disable (pi->hw_if_index, 1);
ip6_sw_interface_enable_disable (pi->hw_if_index, 1);
mpls_sw_interface_enable_disable (&mpls_main, pi->hw_if_index, 1);
@@ -353,7 +364,7 @@ perform_fixed_edits (pg_stream_t * s)
void
pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
pg_stream_t *s;
uword *p;
@@ -418,12 +429,16 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
vec_resize (s->buffer_indices, n);
vec_foreach (bi, s->buffer_indices)
- bi->free_list_index = vlib_buffer_create_free_list (vm, s->buffer_bytes,
- "pg stream %d buffer #%d",
- s - pg->streams,
- 1 + (bi -
- s->
- buffer_indices));
+ {
+ vlib_main_t *vmt;
+ vmt =
+ vlib_num_workers ()? vlib_get_worker_vlib_main (s->worker_index) : vm;
+ bi->free_list_index =
+ vlib_buffer_create_free_list (vmt, s->buffer_bytes,
+ "pg stream %d buffer #%d",
+ s - pg->streams,
+ 1 + (bi - s->buffer_indices));
+ }
}
/* Find an interface to use. */
@@ -447,7 +462,7 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
void
pg_stream_del (pg_main_t * pg, uword index)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
pg_stream_t *s;
pg_buffer_index_t *bi;
@@ -458,7 +473,9 @@ pg_stream_del (pg_main_t * pg, uword index)
vec_foreach (bi, s->buffer_indices)
{
- vlib_buffer_delete_free_list (vm, bi->free_list_index);
+ vlib_buffer_delete_free_list (vlib_num_workers ()?
+ vlib_get_worker_vlib_main (s->worker_index)
+ : vm, bi->free_list_index);
clib_fifo_free (bi->buffer_fifo);
}