summaryrefslogtreecommitdiffstats
path: root/src/plugins/dpdk/buffer.c
diff options
context:
space:
mode:
authorDamjan Marion <damarion@cisco.com>2017-11-10 21:55:45 +0100
committerDave Barach <openvpp@barachs.net>2017-11-13 16:14:50 +0000
commit04f3db3847d242857b9d9d858bcdca538a1be7d7 (patch)
treee0304ad2a3986698d62e9a164f5a0a28565cef01 /src/plugins/dpdk/buffer.c
parent2d8bf304230102a6d9b312b98315418617798175 (diff)
dpdk: introduce AVX512 variants of node functions
Change-Id: If581feca0d51d0420c971801aecdf9250c671b36 Signed-off-by: Damjan Marion <damarion@cisco.com>
Diffstat (limited to 'src/plugins/dpdk/buffer.c')
-rw-r--r--src/plugins/dpdk/buffer.c64
1 files changed, 52 insertions, 12 deletions
diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index 555b1109c19..80c6442f463 100644
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -124,6 +124,7 @@ next:
}
}
+#ifndef CLIB_MULTIARCH_VARIANT
static void
del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
{
@@ -176,6 +177,7 @@ dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
pool_put (bm->buffer_free_list_pool, f);
}
}
+#endif
/* Make sure free list has at least given number of free buffers. */
static uword
@@ -253,10 +255,7 @@ fill_free_list (vlib_main_t * vm,
fl->buffers[f++] = bi2;
fl->buffers[f++] = bi3;
- clib_memcpy (b0, &bt, sizeof (vlib_buffer_t));
- clib_memcpy (b1, &bt, sizeof (vlib_buffer_t));
- clib_memcpy (b2, &bt, sizeof (vlib_buffer_t));
- clib_memcpy (b3, &bt, sizeof (vlib_buffer_t));
+ clib_memcpy64_x4 (b0, b1, b2, b3, &bt);
if (fl->buffer_init_function)
{
@@ -317,7 +316,8 @@ alloc_from_free_list (vlib_main_t * vm,
Returns number actually allocated which will be either zero or
number requested. */
u32
-dpdk_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+CLIB_MULTIARCH_FN (dpdk_buffer_alloc) (vlib_main_t * vm, u32 * buffers,
+ u32 n_buffers)
{
vlib_buffer_main_t *bm = vm->buffer_main;
@@ -330,9 +330,10 @@ dpdk_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
u32
-dpdk_buffer_alloc_from_free_list (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 free_list_index)
+CLIB_MULTIARCH_FN (dpdk_buffer_alloc_from_free_list) (vlib_main_t * vm,
+ u32 * buffers,
+ u32 n_buffers,
+ u32 free_list_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
vlib_buffer_free_list_t *f;
@@ -455,20 +456,23 @@ vlib_buffer_free_inline (vlib_main_t * vm,
}
}
-static void
-dpdk_buffer_free (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+void
+CLIB_MULTIARCH_FN (dpdk_buffer_free) (vlib_main_t * vm, u32 * buffers,
+ u32 n_buffers)
{
vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
1);
}
-static void
-dpdk_buffer_free_no_next (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+void
+CLIB_MULTIARCH_FN (dpdk_buffer_free_no_next) (vlib_main_t * vm, u32 * buffers,
+ u32 n_buffers)
{
vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
0);
}
+#ifndef CLIB_MULTIARCH_VARIANT
static void
dpdk_packet_template_init (vlib_main_t * vm,
void *vt,
@@ -682,6 +686,42 @@ VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
};
/* *INDENT-ON* */
+#if __x86_64__
+vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx512;
+vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx2;
+vlib_buffer_alloc_from_free_list_cb_t __clib_weak
+ dpdk_buffer_alloc_from_free_list_avx512;
+vlib_buffer_alloc_from_free_list_cb_t __clib_weak
+ dpdk_buffer_alloc_from_free_list_avx2;
+vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_cb_avx512;
+vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_cb_avx2;
+vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_cb_avx512;
+vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_cb_avx2;
+
+static void __clib_constructor
+dpdk_input_multiarch_select (void)
+{
+ vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks;
+ if (dpdk_buffer_alloc_avx512 && clib_cpu_supports_avx512f ())
+ {
+ cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx512;
+ cb->vlib_buffer_alloc_from_free_list_cb =
+ dpdk_buffer_alloc_from_free_list_avx512;
+ cb->vlib_buffer_free_cb = dpdk_buffer_free_cb_avx512;
+ cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_cb_avx512;
+ }
+ else if (dpdk_buffer_alloc_avx2 && clib_cpu_supports_avx2 ())
+ {
+ cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx2;
+ cb->vlib_buffer_alloc_from_free_list_cb =
+ dpdk_buffer_alloc_from_free_list_avx2;
+ cb->vlib_buffer_free_cb = dpdk_buffer_free_cb_avx2;
+ cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_cb_avx2;
+ }
+}
+#endif
+#endif
+
/** @endcond */
/*
* fd.io coding-style-patch-verification: ON