diff options
Diffstat (limited to 'src/vnet/classify/vnet_classify.h')
| -rw-r--r-- | src/vnet/classify/vnet_classify.h | 516 |
1 files changed, 278 insertions, 238 deletions
diff --git a/src/vnet/classify/vnet_classify.h b/src/vnet/classify/vnet_classify.h index 4304d33bae8..40628015501 100644 --- a/src/vnet/classify/vnet_classify.h +++ b/src/vnet/classify/vnet_classify.h @@ -28,7 +28,7 @@ #include <vlib/cli.h> #include <vnet/l2/l2_input.h> #include <vnet/l2/feat_bitmap.h> -#include <vnet/api_errno.h> /* for API error numbers */ +#include <vnet/api_errno.h> /* for API error numbers */ #include <vppinfra/error.h> #include <vppinfra/hash.h> @@ -41,7 +41,7 @@ extern vlib_node_registration_t ip6_classify_node; #define CLASSIFY_TRACE 0 #if !defined( __aarch64__) && !defined(__arm__) -#define CLASSIFY_USE_SSE //Allow usage of SSE operations +#define CLASSIFY_USE_SSE //Allow usage of SSE operations #endif #define U32X4_ALIGNED(p) PREDICT_TRUE((((intptr_t)p) & 0xf) == 0) @@ -79,6 +79,7 @@ _(3) \ _(4) \ _(5) +/* *INDENT-OFF* */ typedef CLIB_PACKED(struct _vnet_classify_entry { /* Graph node next index */ u32 next_index; @@ -105,24 +106,28 @@ typedef CLIB_PACKED(struct _vnet_classify_entry { u64 hits; struct _vnet_classify_entry * next_free; }; - + f64 last_heard; /* Must be aligned to a 16-octet boundary */ u32x4 key[0]; }) vnet_classify_entry_t; +/* *INDENT-ON* */ -static inline int vnet_classify_entry_is_free (vnet_classify_entry_t * e) +static inline int +vnet_classify_entry_is_free (vnet_classify_entry_t * e) { return e->flags & VNET_CLASSIFY_ENTRY_FREE; } -static inline int vnet_classify_entry_is_busy (vnet_classify_entry_t * e) +static inline int +vnet_classify_entry_is_busy (vnet_classify_entry_t * e) { return ((e->flags & VNET_CLASSIFY_ENTRY_FREE) == 0); } /* Need these to con the vector allocator */ +/* *INDENT-OFF* */ #define _(size) \ typedef CLIB_PACKED(struct { \ u32 pad0[4]; \ @@ -130,11 +135,15 @@ typedef CLIB_PACKED(struct { \ u32x4 key[size]; \ }) vnet_classify_entry_##size##_t; foreach_size_in_u32x4; +/* *INDENT-ON* */ #undef _ -typedef struct { - union { - struct { +typedef struct +{ + union + { + struct + { u32 offset; u8 linear_search; u8 pad[2]; @@ -144,13 +153,14 @@ typedef struct { }; } vnet_classify_bucket_t; -typedef struct { +typedef struct +{ /* Mask to apply after skipping N vectors */ u32x4 *mask; /* Buckets and entries */ - vnet_classify_bucket_t * buckets; - vnet_classify_entry_t * entries; - + vnet_classify_bucket_t *buckets; + vnet_classify_entry_t *entries; + /* Config parameters */ u32 match_n_vectors; u32 skip_n_vectors; @@ -164,217 +174,219 @@ typedef struct { u32 data_offset; /* Index of next table to try */ u32 next_table_index; - + /* Miss next index, return if next_table_index = 0 */ u32 miss_next_index; - + /* Per-bucket working copies, one per thread */ - vnet_classify_entry_t ** working_copies; + vnet_classify_entry_t **working_copies; int *working_copy_lengths; vnet_classify_bucket_t saved_bucket; - + /* Free entry freelists */ vnet_classify_entry_t **freelists; - u8 * name; - + u8 *name; + /* Private allocation arena, protected by the writer lock */ - void * mheap; - + void *mheap; + /* Writer (only) lock for this table */ - volatile u32 * writer_lock; - + volatile u32 *writer_lock; + } vnet_classify_table_t; -struct _vnet_classify_main { +struct _vnet_classify_main +{ /* Table pool */ - vnet_classify_table_t * tables; - + vnet_classify_table_t *tables; + /* Registered next-index, opaque unformat fcns */ - unformat_function_t ** unformat_l2_next_index_fns; - unformat_function_t ** unformat_ip_next_index_fns; - unformat_function_t ** unformat_acl_next_index_fns; - unformat_function_t ** unformat_policer_next_index_fns; - unformat_function_t ** unformat_opaque_index_fns; + unformat_function_t **unformat_l2_next_index_fns; + unformat_function_t **unformat_ip_next_index_fns; + unformat_function_t **unformat_acl_next_index_fns; + unformat_function_t **unformat_policer_next_index_fns; + unformat_function_t **unformat_opaque_index_fns; /* convenience variables */ - vlib_main_t * vlib_main; - vnet_main_t * vnet_main; + vlib_main_t *vlib_main; + vnet_main_t *vnet_main; }; extern vnet_classify_main_t vnet_classify_main; -u8 * format_classify_table (u8 * s, va_list * args); +u8 *format_classify_table (u8 * s, va_list * args); u64 vnet_classify_hash_packet (vnet_classify_table_t * t, u8 * h); -static inline u64 -vnet_classify_hash_packet_inline (vnet_classify_table_t * t, - u8 * h) +static inline u64 +vnet_classify_hash_packet_inline (vnet_classify_table_t * t, u8 * h) { u32x4 *mask; - union { + union + { u32x4 as_u32x4; u64 as_u64[2]; - } xor_sum __attribute__((aligned(sizeof(u32x4)))); + } xor_sum __attribute__ ((aligned (sizeof (u32x4)))); - ASSERT(t); + ASSERT (t); mask = t->mask; #ifdef CLASSIFY_USE_SSE - if (U32X4_ALIGNED(h)) { //SSE can't handle unaligned data - u32x4 *data = (u32x4 *)h; - xor_sum.as_u32x4 = data[0 + t->skip_n_vectors] & mask[0]; - switch (t->match_n_vectors) - { - case 5: - xor_sum.as_u32x4 ^= data[4 + t->skip_n_vectors] & mask[4]; - /* FALLTHROUGH */ - case 4: - xor_sum.as_u32x4 ^= data[3 + t->skip_n_vectors] & mask[3]; - /* FALLTHROUGH */ - case 3: - xor_sum.as_u32x4 ^= data[2 + t->skip_n_vectors] & mask[2]; - /* FALLTHROUGH */ - case 2: - xor_sum.as_u32x4 ^= data[1 + t->skip_n_vectors] & mask[1]; - /* FALLTHROUGH */ - case 1: - break; - default: - abort(); + if (U32X4_ALIGNED (h)) + { //SSE can't handle unaligned data + u32x4 *data = (u32x4 *) h; + xor_sum.as_u32x4 = data[0 + t->skip_n_vectors] & mask[0]; + switch (t->match_n_vectors) + { + case 5: + xor_sum.as_u32x4 ^= data[4 + t->skip_n_vectors] & mask[4]; + /* FALLTHROUGH */ + case 4: + xor_sum.as_u32x4 ^= data[3 + t->skip_n_vectors] & mask[3]; + /* FALLTHROUGH */ + case 3: + xor_sum.as_u32x4 ^= data[2 + t->skip_n_vectors] & mask[2]; + /* FALLTHROUGH */ + case 2: + xor_sum.as_u32x4 ^= data[1 + t->skip_n_vectors] & mask[1]; + /* FALLTHROUGH */ + case 1: + break; + default: + abort (); + } } - } else + else #endif /* CLASSIFY_USE_SSE */ - { - u32 skip_u64 = t->skip_n_vectors * 2; - u64 *data64 = (u64 *)h; - xor_sum.as_u64[0] = data64[0 + skip_u64] & ((u64 *)mask)[0]; - xor_sum.as_u64[1] = data64[1 + skip_u64] & ((u64 *)mask)[1]; - switch (t->match_n_vectors) { - case 5: - xor_sum.as_u64[0] ^= data64[8 + skip_u64] & ((u64 *)mask)[8]; - xor_sum.as_u64[1] ^= data64[9 + skip_u64] & ((u64 *)mask)[9]; - /* FALLTHROUGH */ - case 4: - xor_sum.as_u64[0] ^= data64[6 + skip_u64] & ((u64 *)mask)[6]; - xor_sum.as_u64[1] ^= data64[7 + skip_u64] & ((u64 *)mask)[7]; - /* FALLTHROUGH */ - case 3: - xor_sum.as_u64[0] ^= data64[4 + skip_u64] & ((u64 *)mask)[4]; - xor_sum.as_u64[1] ^= data64[5 + skip_u64] & ((u64 *)mask)[5]; - /* FALLTHROUGH */ - case 2: - xor_sum.as_u64[0] ^= data64[2 + skip_u64] & ((u64 *)mask)[2]; - xor_sum.as_u64[1] ^= data64[3 + skip_u64] & ((u64 *)mask)[3]; - /* FALLTHROUGH */ - case 1: - break; - - default: - abort(); + u32 skip_u64 = t->skip_n_vectors * 2; + u64 *data64 = (u64 *) h; + xor_sum.as_u64[0] = data64[0 + skip_u64] & ((u64 *) mask)[0]; + xor_sum.as_u64[1] = data64[1 + skip_u64] & ((u64 *) mask)[1]; + switch (t->match_n_vectors) + { + case 5: + xor_sum.as_u64[0] ^= data64[8 + skip_u64] & ((u64 *) mask)[8]; + xor_sum.as_u64[1] ^= data64[9 + skip_u64] & ((u64 *) mask)[9]; + /* FALLTHROUGH */ + case 4: + xor_sum.as_u64[0] ^= data64[6 + skip_u64] & ((u64 *) mask)[6]; + xor_sum.as_u64[1] ^= data64[7 + skip_u64] & ((u64 *) mask)[7]; + /* FALLTHROUGH */ + case 3: + xor_sum.as_u64[0] ^= data64[4 + skip_u64] & ((u64 *) mask)[4]; + xor_sum.as_u64[1] ^= data64[5 + skip_u64] & ((u64 *) mask)[5]; + /* FALLTHROUGH */ + case 2: + xor_sum.as_u64[0] ^= data64[2 + skip_u64] & ((u64 *) mask)[2]; + xor_sum.as_u64[1] ^= data64[3 + skip_u64] & ((u64 *) mask)[3]; + /* FALLTHROUGH */ + case 1: + break; + + default: + abort (); + } } - } - + return clib_xxhash (xor_sum.as_u64[0] ^ xor_sum.as_u64[1]); } -static inline void +static inline void vnet_classify_prefetch_bucket (vnet_classify_table_t * t, u64 hash) { u32 bucket_index; - - ASSERT (is_pow2(t->nbuckets)); - + + ASSERT (is_pow2 (t->nbuckets)); + bucket_index = hash & (t->nbuckets - 1); - - CLIB_PREFETCH(&t->buckets[bucket_index], CLIB_CACHE_LINE_BYTES, LOAD); + + CLIB_PREFETCH (&t->buckets[bucket_index], CLIB_CACHE_LINE_BYTES, LOAD); } -static inline vnet_classify_entry_t * +static inline vnet_classify_entry_t * vnet_classify_get_entry (vnet_classify_table_t * t, uword offset) { - u8 * hp = t->mheap; - u8 * vp = hp + offset; - + u8 *hp = t->mheap; + u8 *vp = hp + offset; + return (void *) vp; } -static inline uword vnet_classify_get_offset (vnet_classify_table_t * t, - vnet_classify_entry_t * v) +static inline uword +vnet_classify_get_offset (vnet_classify_table_t * t, + vnet_classify_entry_t * v) { - u8 * hp, * vp; + u8 *hp, *vp; hp = (u8 *) t->mheap; vp = (u8 *) v; - ASSERT((vp - hp) < 0x100000000ULL); + ASSERT ((vp - hp) < 0x100000000ULL); return vp - hp; } static inline vnet_classify_entry_t * -vnet_classify_entry_at_index (vnet_classify_table_t * t, - vnet_classify_entry_t * e, - u32 index) +vnet_classify_entry_at_index (vnet_classify_table_t * t, + vnet_classify_entry_t * e, u32 index) { - u8 * eu8; + u8 *eu8; - eu8 = (u8 *)e; + eu8 = (u8 *) e; eu8 += index * (sizeof (vnet_classify_entry_t) + - (t->match_n_vectors * sizeof (u32x4))); + (t->match_n_vectors * sizeof (u32x4))); return (vnet_classify_entry_t *) eu8; } static inline void -vnet_classify_prefetch_entry (vnet_classify_table_t * t, - u64 hash) +vnet_classify_prefetch_entry (vnet_classify_table_t * t, u64 hash) { u32 bucket_index; u32 value_index; - vnet_classify_bucket_t * b; - vnet_classify_entry_t * e; + vnet_classify_bucket_t *b; + vnet_classify_entry_t *e; bucket_index = hash & (t->nbuckets - 1); b = &t->buckets[bucket_index]; - + if (b->offset == 0) return; hash >>= t->log2_nbuckets; e = vnet_classify_get_entry (t, b->offset); - value_index = hash & ((1<<b->log2_pages)-1); + value_index = hash & ((1 << b->log2_pages) - 1); e = vnet_classify_entry_at_index (t, e, value_index); - CLIB_PREFETCH(e, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (e, CLIB_CACHE_LINE_BYTES, LOAD); } -vnet_classify_entry_t * -vnet_classify_find_entry (vnet_classify_table_t * t, - u8 * h, u64 hash, f64 now); +vnet_classify_entry_t *vnet_classify_find_entry (vnet_classify_table_t * t, + u8 * h, u64 hash, f64 now); static inline vnet_classify_entry_t * vnet_classify_find_entry_inline (vnet_classify_table_t * t, - u8 * h, u64 hash, f64 now) + u8 * h, u64 hash, f64 now) { - vnet_classify_entry_t * v; + vnet_classify_entry_t *v; u32x4 *mask, *key; - union { + union + { u32x4 as_u32x4; u64 as_u64[2]; - } result __attribute__((aligned(sizeof(u32x4)))); - vnet_classify_bucket_t * b; + } result __attribute__ ((aligned (sizeof (u32x4)))); + vnet_classify_bucket_t *b; u32 value_index; u32 bucket_index; u32 limit; int i; - bucket_index = hash & (t->nbuckets-1); + bucket_index = hash & (t->nbuckets - 1); b = &t->buckets[bucket_index]; mask = t->mask; @@ -384,128 +396,147 @@ vnet_classify_find_entry_inline (vnet_classify_table_t * t, hash >>= t->log2_nbuckets; v = vnet_classify_get_entry (t, b->offset); - value_index = hash & ((1<<b->log2_pages)-1); + value_index = hash & ((1 << b->log2_pages) - 1); limit = t->entries_per_page; if (PREDICT_FALSE (b->linear_search)) { value_index = 0; - limit *= (1<<b->log2_pages); + limit *= (1 << b->log2_pages); } v = vnet_classify_entry_at_index (t, v, value_index); #ifdef CLASSIFY_USE_SSE - if (U32X4_ALIGNED(h)) { - u32x4 *data = (u32x4 *) h; - for (i = 0; i < limit; i++) { - key = v->key; - result.as_u32x4 = (data[0 + t->skip_n_vectors] & mask[0]) ^ key[0]; - switch (t->match_n_vectors) - { - case 5: - result.as_u32x4 |= (data[4 + t->skip_n_vectors] & mask[4]) ^ key[4]; - /* FALLTHROUGH */ - case 4: - result.as_u32x4 |= (data[3 + t->skip_n_vectors] & mask[3]) ^ key[3]; - /* FALLTHROUGH */ - case 3: - result.as_u32x4 |= (data[2 + t->skip_n_vectors] & mask[2]) ^ key[2]; - /* FALLTHROUGH */ - case 2: - result.as_u32x4 |= (data[1 + t->skip_n_vectors] & mask[1]) ^ key[1]; - /* FALLTHROUGH */ - case 1: - break; - default: - abort(); - } - - if (u32x4_zero_byte_mask (result.as_u32x4) == 0xffff) { - if (PREDICT_TRUE(now)) { - v->hits++; - v->last_heard = now; - } - return (v); - } - v = vnet_classify_entry_at_index (t, v, 1); + if (U32X4_ALIGNED (h)) + { + u32x4 *data = (u32x4 *) h; + for (i = 0; i < limit; i++) + { + key = v->key; + result.as_u32x4 = (data[0 + t->skip_n_vectors] & mask[0]) ^ key[0]; + switch (t->match_n_vectors) + { + case 5: + result.as_u32x4 |= + (data[4 + t->skip_n_vectors] & mask[4]) ^ key[4]; + /* FALLTHROUGH */ + case 4: + result.as_u32x4 |= + (data[3 + t->skip_n_vectors] & mask[3]) ^ key[3]; + /* FALLTHROUGH */ + case 3: + result.as_u32x4 |= + (data[2 + t->skip_n_vectors] & mask[2]) ^ key[2]; + /* FALLTHROUGH */ + case 2: + result.as_u32x4 |= + (data[1 + t->skip_n_vectors] & mask[1]) ^ key[1]; + /* FALLTHROUGH */ + case 1: + break; + default: + abort (); + } + + if (u32x4_zero_byte_mask (result.as_u32x4) == 0xffff) + { + if (PREDICT_TRUE (now)) + { + v->hits++; + v->last_heard = now; + } + return (v); + } + v = vnet_classify_entry_at_index (t, v, 1); + } } - } else + else #endif /* CLASSIFY_USE_SSE */ { u32 skip_u64 = t->skip_n_vectors * 2; - u64 *data64 = (u64 *)h; - for (i = 0; i < limit; i++) { - key = v->key; - - result.as_u64[0] = (data64[0 + skip_u64] & ((u64 *)mask)[0]) ^ ((u64 *)key)[0]; - result.as_u64[1] = (data64[1 + skip_u64] & ((u64 *)mask)[1]) ^ ((u64 *)key)[1]; - switch (t->match_n_vectors) - { - case 5: - result.as_u64[0] |= (data64[8 + skip_u64] & ((u64 *)mask)[8]) ^ ((u64 *)key)[8]; - result.as_u64[1] |= (data64[9 + skip_u64] & ((u64 *)mask)[9]) ^ ((u64 *)key)[9]; - /* FALLTHROUGH */ - case 4: - result.as_u64[0] |= (data64[6 + skip_u64] & ((u64 *)mask)[6]) ^ ((u64 *)key)[6]; - result.as_u64[1] |= (data64[7 + skip_u64] & ((u64 *)mask)[7]) ^ ((u64 *)key)[7]; - /* FALLTHROUGH */ - case 3: - result.as_u64[0] |= (data64[4 + skip_u64] & ((u64 *)mask)[4]) ^ ((u64 *)key)[4]; - result.as_u64[1] |= (data64[5 + skip_u64] & ((u64 *)mask)[5]) ^ ((u64 *)key)[5]; - /* FALLTHROUGH */ - case 2: - result.as_u64[0] |= (data64[2 + skip_u64] & ((u64 *)mask)[2]) ^ ((u64 *)key)[2]; - result.as_u64[1] |= (data64[3 + skip_u64] & ((u64 *)mask)[3]) ^ ((u64 *)key)[3]; - /* FALLTHROUGH */ - case 1: - break; - default: - abort(); - } - - if (result.as_u64[0] == 0 && result.as_u64[1] == 0) { - if (PREDICT_TRUE(now)) { - v->hits++; - v->last_heard = now; - } - return (v); - } - - v = vnet_classify_entry_at_index (t, v, 1); - } + u64 *data64 = (u64 *) h; + for (i = 0; i < limit; i++) + { + key = v->key; + + result.as_u64[0] = + (data64[0 + skip_u64] & ((u64 *) mask)[0]) ^ ((u64 *) key)[0]; + result.as_u64[1] = + (data64[1 + skip_u64] & ((u64 *) mask)[1]) ^ ((u64 *) key)[1]; + switch (t->match_n_vectors) + { + case 5: + result.as_u64[0] |= + (data64[8 + skip_u64] & ((u64 *) mask)[8]) ^ ((u64 *) key)[8]; + result.as_u64[1] |= + (data64[9 + skip_u64] & ((u64 *) mask)[9]) ^ ((u64 *) key)[9]; + /* FALLTHROUGH */ + case 4: + result.as_u64[0] |= + (data64[6 + skip_u64] & ((u64 *) mask)[6]) ^ ((u64 *) key)[6]; + result.as_u64[1] |= + (data64[7 + skip_u64] & ((u64 *) mask)[7]) ^ ((u64 *) key)[7]; + /* FALLTHROUGH */ + case 3: + result.as_u64[0] |= + (data64[4 + skip_u64] & ((u64 *) mask)[4]) ^ ((u64 *) key)[4]; + result.as_u64[1] |= + (data64[5 + skip_u64] & ((u64 *) mask)[5]) ^ ((u64 *) key)[5]; + /* FALLTHROUGH */ + case 2: + result.as_u64[0] |= + (data64[2 + skip_u64] & ((u64 *) mask)[2]) ^ ((u64 *) key)[2]; + result.as_u64[1] |= + (data64[3 + skip_u64] & ((u64 *) mask)[3]) ^ ((u64 *) key)[3]; + /* FALLTHROUGH */ + case 1: + break; + default: + abort (); + } + + if (result.as_u64[0] == 0 && result.as_u64[1] == 0) + { + if (PREDICT_TRUE (now)) + { + v->hits++; + v->last_heard = now; + } + return (v); + } + + v = vnet_classify_entry_at_index (t, v, 1); + } } return 0; } -vnet_classify_table_t * -vnet_classify_new_table (vnet_classify_main_t *cm, - u8 * mask, u32 nbuckets, u32 memory_size, - u32 skip_n_vectors, - u32 match_n_vectors); - -int vnet_classify_add_del_session (vnet_classify_main_t * cm, - u32 table_index, - u8 * match, - u32 hit_next_index, - u32 opaque_index, - i32 advance, - u8 action, - u32 metadata, - int is_add); +vnet_classify_table_t *vnet_classify_new_table (vnet_classify_main_t * cm, + u8 * mask, u32 nbuckets, + u32 memory_size, + u32 skip_n_vectors, + u32 match_n_vectors); + +int vnet_classify_add_del_session (vnet_classify_main_t * cm, + u32 table_index, + u8 * match, + u32 hit_next_index, + u32 opaque_index, + i32 advance, + u8 action, u32 metadata, int is_add); int vnet_classify_add_del_table (vnet_classify_main_t * cm, - u8 * mask, - u32 nbuckets, - u32 memory_size, - u32 skip, - u32 match, - u32 next_table_index, - u32 miss_next_index, - u32 * table_index, - u8 current_data_flag, - i16 current_data_offset, - int is_add, - int del_chain); + u8 * mask, + u32 nbuckets, + u32 memory_size, + u32 skip, + u32 match, + u32 next_table_index, + u32 miss_next_index, + u32 * table_index, + u8 current_data_flag, + i16 current_data_offset, + int is_add, int del_chain); unformat_function_t unformat_ip4_mask; unformat_function_t unformat_ip6_mask; @@ -522,18 +553,27 @@ unformat_function_t unformat_vlan_tag; unformat_function_t unformat_l2_match; unformat_function_t unformat_classify_match; -void vnet_classify_register_unformat_ip_next_index_fn -(unformat_function_t * fn); +void vnet_classify_register_unformat_ip_next_index_fn + (unformat_function_t * fn); -void vnet_classify_register_unformat_l2_next_index_fn -(unformat_function_t * fn); +void vnet_classify_register_unformat_l2_next_index_fn + (unformat_function_t * fn); -void vnet_classify_register_unformat_acl_next_index_fn -(unformat_function_t * fn); +void vnet_classify_register_unformat_acl_next_index_fn + (unformat_function_t * fn); -void vnet_classify_register_unformat_policer_next_index_fn -(unformat_function_t * fn); +void vnet_classify_register_unformat_policer_next_index_fn + (unformat_function_t * fn); -void vnet_classify_register_unformat_opaque_index_fn (unformat_function_t * fn); +void vnet_classify_register_unformat_opaque_index_fn (unformat_function_t * + fn); #endif /* __included_vnet_classify_h__ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ |