diff options
| -rw-r--r-- | src/configure.ac | 36 | ||||
| -rw-r--r-- | src/plugins/Makefile.am | 1 | ||||
| -rw-r--r-- | src/plugins/dpdk.am | 32 | ||||
| -rw-r--r-- | src/plugins/dpdk/buffer.c | 64 | ||||
| -rw-r--r-- | src/plugins/dpdk/device/device.c | 30 | ||||
| -rw-r--r-- | src/plugins/dpdk/device/node.c | 64 | ||||
| -rw-r--r-- | src/vlib/buffer.h | 24 | ||||
| -rw-r--r-- | src/vppinfra/clib.h | 5 | ||||
| -rw-r--r-- | src/vppinfra/cpu.h | 7 | ||||
| -rw-r--r-- | src/vppinfra/string.h | 65 |
10 files changed, 259 insertions, 69 deletions
diff --git a/src/configure.ac b/src/configure.ac index ee4985b5471..b5546a7c2e8 100644 --- a/src/configure.ac +++ b/src/configure.ac @@ -119,6 +119,22 @@ AC_DEFUN([DETECT_DPDK_IS_1702_OR_1705], AM_CONDITIONAL(DPDK_IS_1702_OR_1705, test "$dpdk_is_1702_or_1705" = "yes") ]) +# Check if compiler supports specific flag +AC_DEFUN([CC_CHECK_FLAG], +[ + AC_MSG_CHECKING([if $CC supports $1]) + AC_LANG_PUSH([C]) + ac_saved_cflags="$CFLAGS" + CFLAGS="-Werror $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([])], + [c_flag_check=yes], + [c_flag_check=no] +) + AC_MSG_RESULT([$c_flag_check]) + CFLAGS="$ac_saved_cflags" + AC_LANG_POP([C]) +]) + ############################################################################### # configure arguments ############################################################################### @@ -164,6 +180,20 @@ AC_ARG_WITH(pre-data, # Target CPU flags ############################################################################### +# Check if compiler supports march=core-avx2 +CC_CHECK_FLAG("-march=core-avx2") +AS_IF([test "$cc_flag_check" = yes], + [march_core_avx2=yes], + [march_core_avx2=no]) +AM_CONDITIONAL([CC_SUPPORTS_AVX2], [test "$march_core_avx2" = "yes"]) + +# Check if compiler supports march=skylake-avx512 +CC_CHECK_FLAG("-march=skylake-avx512") +AS_IF([test "$cc_flag_check" = yes], + [march_skylake_avx512=yes], + [march_skylake_avx512=no]) +AM_CONDITIONAL([CC_SUPPORTS_AVX512], [test "$march_skylake_avx512" = "yes"]) + AS_CASE([$build_cpu], [x86_64], [CPU_FLAGS="-march=corei7 -mtune=corei7-avx"], [aarch64], [CPU_FLAGS="-march=armv8-a+crc"], @@ -171,6 +201,12 @@ AS_CASE([$build_cpu], ) AC_SUBST([CPU_FLAGS]) +AC_SUBST([CPU_AVX2_FLAGS],"-march=core-avx2 -mtune=core-avx2") +AC_SUBST([CPU_AVX512_FLAGS],"-march=skylake-avx512 -mtune=skylake-avx512") + +AM_CONDITIONAL([CPU_X86_64], [test "$build_cpu" = "x86_64"]) +AM_CONDITIONAL([CPU_AARCH64], [test "$build_cpu" = "aarch64"]) + ############################################################################### # Substitutions and defines ############################################################################### diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am index 97f2f88f821..286fd1f9885 100644 --- a/src/plugins/Makefile.am +++ b/src/plugins/Makefile.am @@ -23,6 +23,7 @@ BUILT_SOURCES = vppplugins_LTLIBRARIES = vppapitestplugins_LTLIBRARIES = noinst_HEADERS = +noinst_LTLIBRARIES = nobase_apiinclude_HEADERS = nobase_include_HEADERS = diff --git a/src/plugins/dpdk.am b/src/plugins/dpdk.am index 905ba2016a3..99abded7cd2 100644 --- a/src/plugins/dpdk.am +++ b/src/plugins/dpdk.am @@ -36,6 +36,7 @@ dpdk_plugin_la_LDFLAGS += -Wl,-lnuma endif dpdk_plugin_la_LDFLAGS += -Wl,-lm,-ldl +dpdk_plugin_la_LIBADD = dpdk_plugin_la_SOURCES = \ dpdk/main.c \ @@ -59,6 +60,37 @@ dpdk_plugin_la_SOURCES = \ API_FILES += dpdk/api/dpdk.api +if CPU_X86_64 +dpdk_multiversioning_files = \ + dpdk/buffer.c \ + dpdk/device/node.c \ + dpdk/device/device.c + +if CC_SUPPORTS_AVX2 +############################################################### +# AVX2 +############################################################### +libdpdk_plugin_avx2_la_SOURCES = $(dpdk_multiversioning_files) +libdpdk_plugin_avx2_la_CFLAGS = \ + $(AM_CFLAGS) @CPU_AVX2_FLAGS@ \ + -DCLIB_MULTIARCH_VARIANT=avx2 +noinst_LTLIBRARIES += libdpdk_plugin_avx2.la +dpdk_plugin_la_LIBADD += libdpdk_plugin_avx2.la +endif + +if CC_SUPPORTS_AVX512 +############################################################### +# AVX512 +############################################################### +libdpdk_plugin_avx512_la_SOURCES = $(dpdk_multiversioning_files) +libdpdk_plugin_avx512_la_CFLAGS = \ + $(AM_CFLAGS) @CPU_AVX512_FLAGS@ \ + -DCLIB_MULTIARCH_VARIANT=avx512 +noinst_LTLIBRARIES += libdpdk_plugin_avx512.la +dpdk_plugin_la_LIBADD += libdpdk_plugin_avx512.la +endif +endif + nobase_include_HEADERS += \ dpdk/device/dpdk.h \ dpdk/api/dpdk_all_api_h.h \ diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index 555b1109c19..80c6442f463 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -124,6 +124,7 @@ next: } } +#ifndef CLIB_MULTIARCH_VARIANT static void del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) { @@ -176,6 +177,7 @@ dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) pool_put (bm->buffer_free_list_pool, f); } } +#endif /* Make sure free list has at least given number of free buffers. */ static uword @@ -253,10 +255,7 @@ fill_free_list (vlib_main_t * vm, fl->buffers[f++] = bi2; fl->buffers[f++] = bi3; - clib_memcpy (b0, &bt, sizeof (vlib_buffer_t)); - clib_memcpy (b1, &bt, sizeof (vlib_buffer_t)); - clib_memcpy (b2, &bt, sizeof (vlib_buffer_t)); - clib_memcpy (b3, &bt, sizeof (vlib_buffer_t)); + clib_memcpy64_x4 (b0, b1, b2, b3, &bt); if (fl->buffer_init_function) { @@ -317,7 +316,8 @@ alloc_from_free_list (vlib_main_t * vm, Returns number actually allocated which will be either zero or number requested. */ u32 -dpdk_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +CLIB_MULTIARCH_FN (dpdk_buffer_alloc) (vlib_main_t * vm, u32 * buffers, + u32 n_buffers) { vlib_buffer_main_t *bm = vm->buffer_main; @@ -330,9 +330,10 @@ dpdk_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) u32 -dpdk_buffer_alloc_from_free_list (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, u32 free_list_index) +CLIB_MULTIARCH_FN (dpdk_buffer_alloc_from_free_list) (vlib_main_t * vm, + u32 * buffers, + u32 n_buffers, + u32 free_list_index) { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *f; @@ -455,20 +456,23 @@ vlib_buffer_free_inline (vlib_main_t * vm, } } -static void -dpdk_buffer_free (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +void +CLIB_MULTIARCH_FN (dpdk_buffer_free) (vlib_main_t * vm, u32 * buffers, + u32 n_buffers) { vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ 1); } -static void -dpdk_buffer_free_no_next (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +void +CLIB_MULTIARCH_FN (dpdk_buffer_free_no_next) (vlib_main_t * vm, u32 * buffers, + u32 n_buffers) { vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ 0); } +#ifndef CLIB_MULTIARCH_VARIANT static void dpdk_packet_template_init (vlib_main_t * vm, void *vt, @@ -682,6 +686,42 @@ VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = { }; /* *INDENT-ON* */ +#if __x86_64__ +vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx512; +vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx2; +vlib_buffer_alloc_from_free_list_cb_t __clib_weak + dpdk_buffer_alloc_from_free_list_avx512; +vlib_buffer_alloc_from_free_list_cb_t __clib_weak + dpdk_buffer_alloc_from_free_list_avx2; +vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_cb_avx512; +vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_cb_avx2; +vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_cb_avx512; +vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_cb_avx2; + +static void __clib_constructor +dpdk_input_multiarch_select (void) +{ + vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks; + if (dpdk_buffer_alloc_avx512 && clib_cpu_supports_avx512f ()) + { + cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx512; + cb->vlib_buffer_alloc_from_free_list_cb = + dpdk_buffer_alloc_from_free_list_avx512; + cb->vlib_buffer_free_cb = dpdk_buffer_free_cb_avx512; + cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_cb_avx512; + } + else if (dpdk_buffer_alloc_avx2 && clib_cpu_supports_avx2 ()) + { + cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx2; + cb->vlib_buffer_alloc_from_free_list_cb = + dpdk_buffer_alloc_from_free_list_avx2; + cb->vlib_buffer_free_cb = dpdk_buffer_free_cb_avx2; + cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_cb_avx2; + } +} +#endif +#endif + /** @endcond */ /* * fd.io coding-style-patch-verification: ON diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index aa134327373..987596ead9b 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -38,6 +38,7 @@ typedef enum DPDK_TX_FUNC_N_ERROR, } dpdk_tx_func_error_t; +#ifndef CLIB_MULTIARCH_VARIANT static char *dpdk_tx_func_error_strings[] = { #define _(n,s) s, foreach_dpdk_tx_func_error @@ -65,8 +66,9 @@ dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address) return NULL; } } +#endif -struct rte_mbuf * +static struct rte_mbuf * dpdk_replicate_packet_mb (vlib_buffer_t * b) { dpdk_main_t *dm = &dpdk_main; @@ -368,9 +370,10 @@ dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b, * rte_mbuf pointers. It then passes this vector to tx_burst_vector_internal * which calls the dpdk tx_burst function. */ -static uword -dpdk_interface_tx (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * f) +uword +CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * f) { dpdk_main_t *dm = &dpdk_main; vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; @@ -632,6 +635,7 @@ dpdk_interface_tx (vlib_main_t * vm, return tx_pkts; } +#ifndef CLIB_MULTIARCH_VARIANT static void dpdk_clear_hw_interface_counters (u32 instance) { @@ -789,12 +793,25 @@ VNET_DEVICE_CLASS (dpdk_device_class) = { .rx_redirect_to_node = dpdk_set_interface_next_node, .mac_addr_change_function = dpdk_set_mac_address, }; - -VLIB_DEVICE_TX_FUNCTION_MULTIARCH (dpdk_device_class, dpdk_interface_tx) /* *INDENT-ON* */ +#if __x86_64__ +vlib_node_function_t __clib_weak dpdk_interface_tx_avx512; +vlib_node_function_t __clib_weak dpdk_interface_tx_avx2; +static void __clib_constructor +dpdk_interface_tx_multiarch_select (void) +{ + if (dpdk_interface_tx_avx512 && clib_cpu_supports_avx512f ()) + dpdk_device_class.tx_function = dpdk_interface_tx_avx512; + else if (dpdk_interface_tx_avx2 && clib_cpu_supports_avx2 ()) + dpdk_device_class.tx_function = dpdk_interface_tx_avx2; +} +#endif +#endif + #define UP_DOWN_FLAG_EVENT 1 +#ifndef CLIB_MULTIARCH_VARIANT uword admin_up_down_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) @@ -846,6 +863,7 @@ VLIB_REGISTER_NODE (admin_up_down_process_node,static) = { .process_log2_n_stack_bytes = 17, // 256KB }; /* *INDENT-ON* */ +#endif /* * fd.io coding-style-patch-verification: ON diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index 82978216817..1240b8d01a7 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -28,11 +28,13 @@ #include <dpdk/device/dpdk_priv.h> +#ifndef CLIB_MULTIARCH_VARIANT static char *dpdk_error_strings[] = { #define _(n,s) s, foreach_dpdk_error #undef _ }; +#endif always_inline int vlib_buffer_is_ip4 (vlib_buffer_t * b) @@ -259,43 +261,6 @@ dpdk_prefetch_ethertype (struct rte_mbuf *mb) CLIB_CACHE_LINE_BYTES, LOAD); } - -/* - This function should fill 1st cacheline of vlib_buffer_t metadata with data - from buffer template. Instead of filling field by field, we construct - template and then use 128/256 bit vector instruction to copy data. - This code first loads whole cacheline into 4 128-bit registers (xmm) - or two 256 bit registers (ymm) and then stores data into all 4 buffers - efectively saving on register load operations. -*/ - -static_always_inline void -dpdk_buffer_init_from_template (void *d0, void *d1, void *d2, void *d3, - void *s) -{ -#if defined(CLIB_HAVE_VEC128) - int i; - for (i = 0; i < 2; i++) - { - *(u8x32 *) (((u8 *) d0) + i * 32) = - *(u8x32 *) (((u8 *) d1) + i * 32) = - *(u8x32 *) (((u8 *) d2) + i * 32) = - *(u8x32 *) (((u8 *) d3) + i * 32) = *(u8x32 *) (((u8 *) s) + i * 32); - } -#elif defined(CLIB_HAVE_VEC64) - int i; - for (i = 0; i < 4; i++) - { - *(u8x16 *) (((u8 *) d0) + i * 16) = - *(u8x16 *) (((u8 *) d1) + i * 16) = - *(u8x16 *) (((u8 *) d2) + i * 16) = - *(u8x16 *) (((u8 *) d3) + i * 16) = *(u8x16 *) (((u8 *) s) + i * 16); - } -#else -#error "Either CLIB_HAVE_VEC128 or CLIB_HAVE_VEC64 has to be defined" -#endif -} - /* * This function is used when there are no worker threads. * The main thread performs IO and forwards the packets. @@ -401,7 +366,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, b2 = vlib_buffer_from_rte_mbuf (mb2); b3 = vlib_buffer_from_rte_mbuf (mb3); - dpdk_buffer_init_from_template (b0, b1, b2, b3, bt); + clib_memcpy64_x4 (b0, b1, b2, b3, bt); dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 9]); dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 5]); @@ -647,8 +612,9 @@ poll_rate_limit (dpdk_main_t * dm) <code>xd->per_interface_next_index</code> */ -static uword -dpdk_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f) +uword +CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * f) { dpdk_main_t *dm = &dpdk_main; dpdk_device_t *xd; @@ -678,6 +644,7 @@ dpdk_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f) return n_rx_packets; } +#ifndef CLIB_MULTIARCH_VARIANT /* *INDENT-OFF* */ VLIB_REGISTER_NODE (dpdk_input_node) = { .function = dpdk_input, @@ -694,10 +661,23 @@ VLIB_REGISTER_NODE (dpdk_input_node) = { .n_errors = DPDK_N_ERROR, .error_strings = dpdk_error_strings, }; - -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_input_node, dpdk_input); /* *INDENT-ON* */ +vlib_node_function_t __clib_weak dpdk_input_avx512; +vlib_node_function_t __clib_weak dpdk_input_avx2; + +#if __x86_64__ +static void __clib_constructor +dpdk_input_multiarch_select (void) +{ + if (dpdk_input_avx512 && clib_cpu_supports_avx512f ()) + dpdk_input_node.function = dpdk_input_avx512; + else if (dpdk_input_avx2 && clib_cpu_supports_avx2 ()) + dpdk_input_node.function = dpdk_input_avx2; +} +#endif +#endif + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index 6170323c815..21bed48ac14 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -380,17 +380,23 @@ typedef struct vlib_buffer_free_list_t uword buffer_init_function_opaque; } __attribute__ ((aligned (16))) vlib_buffer_free_list_t; +typedef u32 (vlib_buffer_alloc_cb_t) (struct vlib_main_t * vm, u32 * buffers, + u32 n_buffers); +typedef u32 (vlib_buffer_alloc_from_free_list_cb_t) (struct vlib_main_t * vm, + u32 * buffers, + u32 n_buffers, + u32 free_list_index); +typedef void (vlib_buffer_free_cb_t) (struct vlib_main_t * vm, u32 * buffers, + u32 n_buffers); +typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm, + u32 * buffers, u32 n_buffers); + typedef struct { - u32 (*vlib_buffer_alloc_cb) (struct vlib_main_t * vm, u32 * buffers, - u32 n_buffers); - u32 (*vlib_buffer_alloc_from_free_list_cb) (struct vlib_main_t * vm, - u32 * buffers, u32 n_buffers, - u32 free_list_index); - void (*vlib_buffer_free_cb) (struct vlib_main_t * vm, u32 * buffers, - u32 n_buffers); - void (*vlib_buffer_free_no_next_cb) (struct vlib_main_t * vm, u32 * buffers, - u32 n_buffers); + vlib_buffer_alloc_cb_t *vlib_buffer_alloc_cb; + vlib_buffer_alloc_from_free_list_cb_t *vlib_buffer_alloc_from_free_list_cb; + vlib_buffer_free_cb_t *vlib_buffer_free_cb; + vlib_buffer_free_no_next_cb_t *vlib_buffer_free_no_next_cb; void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t, void *packet_data, uword n_packet_data_bytes, diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h index fbb2a21c6b9..33db3b203f9 100644 --- a/src/vppinfra/clib.h +++ b/src/vppinfra/clib.h @@ -78,6 +78,11 @@ #define CLIB_PACKED(x) x __attribute__ ((packed)) #define CLIB_UNUSED(x) x __attribute__ ((unused)) +#define __clib_unused __attribute__ ((unused)) +#define __clib_weak __attribute__ ((weak)) +#define __clib_packed __attribute__ ((packed)) +#define __clib_constructor __attribute__ ((constructor)) + #define never_inline __attribute__ ((__noinline__)) #if CLIB_DEBUG > 0 diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h index 9c149f3fa2a..75b01e606f3 100644 --- a/src/vppinfra/cpu.h +++ b/src/vppinfra/cpu.h @@ -51,6 +51,13 @@ return & fn; \ } +#ifdef CLIB_MULTIARCH_VARIANT +#define __CLIB_MULTIARCH_FN(a,b) a##_##b +#define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b) +#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MULTIARCH_VARIANT) +#else +#define CLIB_MULTIARCH_FN(fn) fn +#endif #define foreach_x86_64_flags \ _ (sse3, 1, ecx, 0) \ diff --git a/src/vppinfra/string.h b/src/vppinfra/string.h index 69a99a3f0ce..914f6a7bbc4 100644 --- a/src/vppinfra/string.h +++ b/src/vppinfra/string.h @@ -72,6 +72,71 @@ void clib_memswap (void *_a, void *_b, uword bytes); #define clib_memcpy(a,b,c) memcpy(a,b,c) #endif +/* + * Copy 64 bytes of data to 4 destinations + * this function is typically used in quad-loop case when whole cacheline + * needs to be copied to 4 different places. First it reads whole cacheline + * to 1/2/4 SIMD registers and then it writes data to 4 destinations. + */ + +static_always_inline void +clib_memcpy64_x4 (void *d0, void *d1, void *d2, void *d3, void *s) +{ +#if defined (CLIB_HAVE_VEC512) + u8x64 __attribute__ ((aligned (1))) r0 = *(((u8x64 *) s) + 0); + + *(((u8x64 *) d0) + 0) = r0; + *(((u8x64 *) d1) + 0) = r0; + *(((u8x64 *) d2) + 0) = r0; + *(((u8x64 *) d3) + 0) = r0; +#elif defined (CLIB_HAVE_VEC256) + u8x32 __attribute__ ((aligned (1))) r0 = *(((u8x32 *) s) + 0); + u8x32 __attribute__ ((aligned (1))) r1 = *(((u8x32 *) s) + 1); + + *(((u8x32 *) d0) + 0) = r0; + *(((u8x32 *) d0) + 1) = r1; + + *(((u8x32 *) d1) + 0) = r0; + *(((u8x32 *) d1) + 1) = r1; + + *(((u8x32 *) d2) + 0) = r0; + *(((u8x32 *) d2) + 1) = r1; + + *(((u8x32 *) d3) + 0) = r0; + *(((u8x32 *) d3) + 1) = r1; +#elif defined (CLIB_HAVE_VEC128) + u8x16 __attribute__ ((aligned (1))) r0 = *(((u8x16 *) s) + 0); + u8x16 __attribute__ ((aligned (1))) r1 = *(((u8x16 *) s) + 1); + u8x16 __attribute__ ((aligned (1))) r2 = *(((u8x16 *) s) + 3); + u8x16 __attribute__ ((aligned (1))) r3 = *(((u8x16 *) s) + 4); + + *(((u8x16 *) d0) + 0) = r0; + *(((u8x16 *) d0) + 1) = r1; + *(((u8x16 *) d0) + 2) = r2; + *(((u8x16 *) d0) + 3) = r3; + + *(((u8x16 *) d1) + 0) = r0; + *(((u8x16 *) d1) + 1) = r1; + *(((u8x16 *) d1) + 2) = r2; + *(((u8x16 *) d1) + 3) = r3; + + *(((u8x16 *) d2) + 0) = r0; + *(((u8x16 *) d2) + 1) = r1; + *(((u8x16 *) d2) + 2) = r2; + *(((u8x16 *) d2) + 3) = r3; + + *(((u8x16 *) d3) + 0) = r0; + *(((u8x16 *) d3) + 1) = r1; + *(((u8x16 *) d3) + 2) = r2; + *(((u8x16 *) d3) + 3) = r3; +#else + clib_memcpy (d0, s, 64); + clib_memcpy (d1, s, 64); + clib_memcpy (d2, s, 64); + clib_memcpy (d3, s, 64); +#endif +} + #endif /* included_clib_string_h */ /* |