summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--src/vlib/buffer_funcs.c23
-rw-r--r--src/vlib/node_funcs.h85
-rw-r--r--src/vnet/interface_output.c8
3 files changed, 97 insertions, 19 deletions
diff --git a/src/vlib/buffer_funcs.c b/src/vlib/buffer_funcs.c
index a661370a141..32c2d1b8a2f 100644
--- a/src/vlib/buffer_funcs.c
+++ b/src/vlib/buffer_funcs.c
@@ -8,11 +8,11 @@
#include <vppinfra/vector/compress.h>
static_always_inline u32
-enqueue_one (vlib_main_t *vm, vlib_node_runtime_t *node, u64 *used_elt_bmp,
- u16 next_index, u32 *buffers, u16 *nexts, u32 n_buffers,
- u32 n_left, u32 *tmp)
+enqueue_one (vlib_main_t *vm, vlib_node_runtime_t *node,
+ vlib_frame_bitmap_t used_elt_bmp, u16 next_index, u32 *buffers,
+ u16 *nexts, u32 n_buffers, u32 n_left, u32 *tmp)
{
- u64 match_bmp[VLIB_FRAME_SIZE / 64];
+ vlib_frame_bitmap_t match_bmp;
vlib_frame_t *f;
u32 n_extracted, n_free;
u32 *to;
@@ -29,11 +29,8 @@ enqueue_one (vlib_main_t *vm, vlib_node_runtime_t *node, u64 *used_elt_bmp,
to = tmp;
clib_mask_compare_u16 (next_index, nexts, match_bmp, n_buffers);
-
n_extracted = clib_compress_u32 (to, buffers, match_bmp, n_buffers);
-
- for (int i = 0; i < ARRAY_LEN (match_bmp); i++)
- used_elt_bmp[i] |= match_bmp[i];
+ vlib_frame_bitmap_or (used_elt_bmp, match_bmp);
if (to != tmp)
{
@@ -77,7 +74,7 @@ CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_next_fn)
while (count >= VLIB_FRAME_SIZE)
{
- u64 used_elt_bmp[VLIB_FRAME_SIZE / 64] = {};
+ vlib_frame_bitmap_t used_elt_bmp = {};
n_left = VLIB_FRAME_SIZE;
u32 off = 0;
@@ -106,7 +103,7 @@ CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_next_fn)
if (count)
{
- u64 used_elt_bmp[VLIB_FRAME_SIZE / 64] = {};
+ vlib_frame_bitmap_t used_elt_bmp = {};
next_index = nexts[0];
n_left = count;
u32 off = 0;
@@ -208,8 +205,7 @@ vlib_buffer_enqueue_to_thread_inline (vlib_main_t *vm,
u32 n_packets, int drop_on_congestion)
{
u32 drop_list[VLIB_FRAME_SIZE], n_drop = 0;
- u64 used_elts[VLIB_FRAME_SIZE / 64] = {};
- u64 mask[VLIB_FRAME_SIZE / 64];
+ vlib_frame_bitmap_t mask, used_elts = {};
vlib_frame_queue_elt_t *hf = 0;
u16 thread_index;
u32 n_comp, off = 0, n_left = n_packets;
@@ -238,8 +234,7 @@ more:
if (n_left)
{
- for (int i = 0; i < ARRAY_LEN (used_elts); i++)
- used_elts[i] |= mask[i];
+ vlib_frame_bitmap_or (used_elts, mask);
while (PREDICT_FALSE (used_elts[off] == ~0))
{
diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index 46db46dd1e0..a9101f5d16a 100644
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -1279,6 +1279,91 @@ vlib_node_function_t *
vlib_node_get_preferred_node_fn_variant (vlib_main_t *vm,
vlib_node_fn_registration_t *regs);
+/*
+ * vlib_frame_bitmap functions
+ */
+
+#define VLIB_FRAME_BITMAP_N_UWORDS \
+ (((VLIB_FRAME_SIZE + uword_bits - 1) & ~(uword_bits - 1)) / uword_bits)
+
+typedef uword vlib_frame_bitmap_t[VLIB_FRAME_BITMAP_N_UWORDS];
+
+static_always_inline void
+vlib_frame_bitmap_init (uword *bmp, u32 n_first_bits_set)
+{
+ u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
+ while (n_first_bits_set >= (sizeof (uword) * 8) && n_left)
+ {
+ bmp++[0] = ~0;
+ n_first_bits_set -= sizeof (uword) * 8;
+ n_left--;
+ }
+
+ if (n_first_bits_set && n_left)
+ {
+ bmp++[0] = pow2_mask (n_first_bits_set);
+ n_left--;
+ }
+
+ while (n_left--)
+ bmp++[0] = 0;
+}
+
+static_always_inline void
+vlib_frame_bitmap_clear (uword *bmp)
+{
+ u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
+ while (n_left--)
+ bmp++[0] = 0;
+}
+
+static_always_inline void
+vlib_frame_bitmap_xor (uword *bmp, uword *bmp2)
+{
+ u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
+ while (n_left--)
+ bmp++[0] ^= bmp2++[0];
+}
+
+static_always_inline void
+vlib_frame_bitmap_or (uword *bmp, uword *bmp2)
+{
+ u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
+ while (n_left--)
+ bmp++[0] |= bmp2++[0];
+}
+
+static_always_inline u32
+vlib_frame_bitmap_count_set_bits (uword *bmp)
+{
+ u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS;
+ u32 count = 0;
+ while (n_left--)
+ count += count_set_bits (bmp++[0]);
+ return count;
+}
+
+static_always_inline int
+vlib_frame_bitmap_find_first_set (uword *bmp)
+{
+ uword *b = bmp;
+ while (b[0] == 0)
+ {
+ ASSERT (b - bmp < VLIB_FRAME_BITMAP_N_UWORDS);
+ b++;
+ }
+
+ return (b - bmp) * uword_bits + get_lowest_set_bit_index (b[0]);
+}
+
+#define foreach_vlib_frame_bitmap_set_bit_index(i, v) \
+ for (uword _off = 0; _off < ARRAY_LEN (v); _off++) \
+ for (uword _tmp = \
+ (v[_off]) + 0 * (uword) (i = _off * uword_bits + \
+ get_lowest_set_bit_index (v[_off])); \
+ _tmp; i = _off * uword_bits + get_lowest_set_bit_index ( \
+ _tmp = clear_lowest_set_bit (_tmp)))
+
#endif /* included_vlib_node_funcs_h */
/*
diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c
index cf64eeed093..6d5697deac4 100644
--- a/src/vnet/interface_output.c
+++ b/src/vnet/interface_output.c
@@ -405,7 +405,7 @@ enqueue_one_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ppqi,
u32 n_vectors, u32 n_left, u32 next_index)
{
u32 tmp[VLIB_FRAME_SIZE];
- u64 mask[VLIB_FRAME_SIZE / 64] = {};
+ vlib_frame_bitmap_t mask = {};
vlib_frame_t *f;
vnet_hw_if_tx_frame_t *tf;
u32 *to;
@@ -1259,8 +1259,7 @@ VLIB_NODE_FN (vnet_interface_output_arc_end_node)
vnet_hw_interface_t *hi;
vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
u32 sw_if_indices[VLIB_FRAME_SIZE], *sw_if_index = sw_if_indices;
- u64 used_elts[VLIB_FRAME_SIZE / 64] = {};
- u64 mask[VLIB_FRAME_SIZE / 64] = {};
+ vlib_frame_bitmap_t used_elts = {}, mask = {};
u32 *tmp, *from, n_left, n_comp, n_p_comp, swif, off;
u16 next_index;
void *ptr[VLIB_FRAME_SIZE], **p = ptr;
@@ -1350,8 +1349,7 @@ drop:
if (n_left)
{
/* store comparison mask so we can find next unused element */
- for (int i = 0; i < ARRAY_LEN (used_elts); i++)
- used_elts[i] |= mask[i];
+ vlib_frame_bitmap_or (used_elts, mask);
/* fine first unused sw_if_index by scanning trough used_elts bitmap */
while (PREDICT_FALSE (used_elts[off] == ~0))