diff options
| -rw-r--r-- | src/vnet/ethernet/ethernet.h | 57 | ||||
| -rwxr-xr-x | src/vnet/ethernet/node.c | 27 | ||||
| -rw-r--r-- | src/vppinfra/vector_sse2.h | 25 |
3 files changed, 68 insertions, 41 deletions
diff --git a/src/vnet/ethernet/ethernet.h b/src/vnet/ethernet/ethernet.h index 80c6ccd3c3b..389bc1b3bf4 100644 --- a/src/vnet/ethernet/ethernet.h +++ b/src/vnet/ethernet/ethernet.h @@ -67,25 +67,23 @@ ethernet_mac_address_is_zero (u8 * mac) return ((*((u32 *) mac) == 0) && (*((u16 *) (mac + 4)) == 0)); } +static const u16x8 tagged_ethertypes = { + (u16) ETHERNET_TYPE_VLAN, + (u16) ETHERNET_TYPE_DOT1AD, + (u16) ETHERNET_TYPE_VLAN_9100, + (u16) ETHERNET_TYPE_VLAN_9200, + /* duplicate last one to fill register */ + (u16) ETHERNET_TYPE_VLAN_9200, + (u16) ETHERNET_TYPE_VLAN_9200, + (u16) ETHERNET_TYPE_VLAN_9200, + (u16) ETHERNET_TYPE_VLAN_9200 +}; + static_always_inline int ethernet_frame_is_tagged (u16 type) { -#if __SSE4_2__ - const __m128i ethertype_mask = _mm_set_epi16 ((u16) ETHERNET_TYPE_VLAN, - (u16) ETHERNET_TYPE_DOT1AD, - (u16) ETHERNET_TYPE_VLAN_9100, - (u16) ETHERNET_TYPE_VLAN_9200, - /* duplicate last one to - fill register */ - (u16) ETHERNET_TYPE_VLAN_9200, - (u16) ETHERNET_TYPE_VLAN_9200, - (u16) ETHERNET_TYPE_VLAN_9200, - (u16) - ETHERNET_TYPE_VLAN_9200); - - __m128i r = _mm_set1_epi16 (type); - r = _mm_cmpeq_epi16 (ethertype_mask, r); - return !_mm_test_all_zeros (r, r); +#ifdef CLIB_HAVE_VEC128 + return !u16x8_is_all_zero (tagged_ethertypes == u16x8_splat (type)); #else if ((type == ETHERNET_TYPE_VLAN) || (type == ETHERNET_TYPE_DOT1AD) || @@ -95,6 +93,33 @@ ethernet_frame_is_tagged (u16 type) return 0; } +static_always_inline int +ethernet_frame_is_any_tagged_x2 (u16 type0, u16 type1) +{ +#ifdef CLIB_HAVE_VEC128 + u16x8 r0 = (tagged_ethertypes == u16x8_splat (type0)); + u16x8 r1 = (tagged_ethertypes == u16x8_splat (type1)); + return !u16x8_is_all_zero (r0 | r1); +#else + return ethernet_frame_is_tagged (type0) || ethernet_frame_is_tagged (type1); +#endif +} + +static_always_inline int +ethernet_frame_is_any_tagged_x4 (u16 type0, u16 type1, u16 type2, u16 type3) +{ +#ifdef CLIB_HAVE_VEC128 + u16x8 r0 = (tagged_ethertypes == u16x8_splat (type0)); + u16x8 r1 = (tagged_ethertypes == u16x8_splat (type1)); + u16x8 r2 = (tagged_ethertypes == u16x8_splat (type2)); + u16x8 r3 = (tagged_ethertypes == u16x8_splat (type3)); + return !u16x8_is_all_zero (r0 | r1 | r2 | r3); +#else + return ethernet_frame_is_tagged (type0) || ethernet_frame_is_tagged (type1) + || ethernet_frame_is_tagged (type2) || ethernet_frame_is_tagged (type3); +#endif +} + /* Max. sized ethernet/vlan header for parsing. */ typedef struct { diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index 3b85483732c..d13af57a1df 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -285,30 +285,6 @@ determine_next_node (ethernet_main_t * em, } } -static_always_inline int -ethernet_frame_is_any_tagged (u16 type0, u16 type1) -{ -#if __SSE4_2__ - const __m128i ethertype_mask = _mm_set_epi16 ((u16) ETHERNET_TYPE_VLAN, - (u16) ETHERNET_TYPE_DOT1AD, - (u16) ETHERNET_TYPE_VLAN_9100, - (u16) ETHERNET_TYPE_VLAN_9200, - /* duplicate for type1 */ - (u16) ETHERNET_TYPE_VLAN, - (u16) ETHERNET_TYPE_DOT1AD, - (u16) ETHERNET_TYPE_VLAN_9100, - (u16) - ETHERNET_TYPE_VLAN_9200); - - __m128i r = - _mm_set_epi16 (type0, type0, type0, type0, type1, type1, type1, type1); - r = _mm_cmpeq_epi16 (ethertype_mask, r); - return !_mm_test_all_zeros (r, r); -#else - return ethernet_frame_is_tagged (type0) || ethernet_frame_is_tagged (type1); -#endif -} - static_always_inline uword ethernet_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -401,7 +377,8 @@ ethernet_input_inline (vlib_main_t * vm, /* Speed-path for the untagged case */ if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET - && !ethernet_frame_is_any_tagged (type0, type1))) + && !ethernet_frame_is_any_tagged_x2 (type0, + type1))) { main_intf_t *intf0; subint_config_t *subint0; diff --git a/src/vppinfra/vector_sse2.h b/src/vppinfra/vector_sse2.h index 6830d5c6104..ce1bb81c93f 100644 --- a/src/vppinfra/vector_sse2.h +++ b/src/vppinfra/vector_sse2.h @@ -545,6 +545,31 @@ u32x4_is_zero (u32x4 x) return u32x4_is_equal (x, zero); } + +always_inline int +u8x16_is_all_zero (u8x16 x) +{ + return _mm_testz_si128 ((__m128i) x, (__m128i) x); +} + +always_inline int +u16x8_is_all_zero (u16x8 x) +{ + return _mm_testz_si128 ((__m128i) x, (__m128i) x); +} + +always_inline int +u32x4_is_all_zero (u32x4 x) +{ + return _mm_testz_si128 ((__m128i) x, (__m128i) x); +} + +always_inline int +u64x2_is_all_zero (u64x2 x) +{ + return _mm_testz_si128 ((__m128i) x, (__m128i) x); +} + #define u32x4_select(A,MASK) \ ({ \ u32x4 _x, _y; \ |