diff options
| -rw-r--r-- | src/vnet/ipsec/esp_decrypt.c | 3 | ||||
| -rw-r--r-- | src/vnet/ipsec/esp_encrypt.c | 7 | ||||
| -rw-r--r-- | src/vnet/ipsec/ipsec_sa.h | 60 |
3 files changed, 44 insertions, 26 deletions
diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c index 75092cc9581..ff9fc0c2d37 100644 --- a/src/vnet/ipsec/esp_decrypt.c +++ b/src/vnet/ipsec/esp_decrypt.c @@ -1096,6 +1096,9 @@ esp_decrypt_inline (vlib_main_t * vm, current_sa_index = vnet_buffer (b[0])->ipsec.sad_index; sa0 = pool_elt_at_index (im->sad, current_sa_index); + + /* fetch the second cacheline ASAP */ + CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD); cpd.icv_sz = sa0->integ_icv_size; cpd.iv_sz = sa0->crypto_iv_size; cpd.flags = sa0->flags; diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c index c0773e2963f..591ec260699 100644 --- a/src/vnet/ipsec/esp_encrypt.c +++ b/src/vnet/ipsec/esp_encrypt.c @@ -624,6 +624,9 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); p -= CLIB_CACHE_LINE_BYTES; CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); + /* speculate that the trailer goes in the first buffer */ + CLIB_PREFETCH (vlib_buffer_get_tail (b[1]), + CLIB_CACHE_LINE_BYTES, LOAD); } if (is_tun) @@ -646,6 +649,10 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, current_sa_packets = current_sa_bytes = 0; sa0 = pool_elt_at_index (im->sad, sa_index0); + + /* fetch the second cacheline ASAP */ + CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD); + current_sa_index = sa_index0; spi = clib_net_to_host_u32 (sa0->spi); esp_align = sa0->esp_block_align; diff --git a/src/vnet/ipsec/ipsec_sa.h b/src/vnet/ipsec/ipsec_sa.h index 5d65238b736..02f5eaf03a0 100644 --- a/src/vnet/ipsec/ipsec_sa.h +++ b/src/vnet/ipsec/ipsec_sa.h @@ -40,7 +40,7 @@ typedef enum foreach_ipsec_crypto_alg #undef _ IPSEC_CRYPTO_N_ALG, -} ipsec_crypto_alg_t; +} __clib_packed ipsec_crypto_alg_t; #define IPSEC_CRYPTO_ALG_IS_GCM(_alg) \ (((_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) || \ @@ -62,13 +62,13 @@ typedef enum foreach_ipsec_integ_alg #undef _ IPSEC_INTEG_N_ALG, -} ipsec_integ_alg_t; +} __clib_packed ipsec_integ_alg_t; typedef enum { IPSEC_PROTOCOL_AH = 0, IPSEC_PROTOCOL_ESP = 1 -} ipsec_protocol_t; +} __clib_packed ipsec_protocol_t; #define IPSEC_KEY_MAX_LEN 128 typedef struct ipsec_key_t_ @@ -149,9 +149,9 @@ typedef struct u64 crypto_op_data; }; - /* data accessed by dataplane code should be above this comment */ CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); + u64 gcm_iv_counter; union { ip4_header_t ip4_hdr; @@ -159,31 +159,16 @@ typedef struct }; udp_header_t udp_hdr; - fib_node_t node; - u32 id; - u32 stat_index; - ipsec_protocol_t protocol; - - ipsec_crypto_alg_t crypto_alg; - ipsec_key_t crypto_key; - vnet_crypto_alg_t crypto_calg; - - ipsec_integ_alg_t integ_alg; - ipsec_key_t integ_key; - vnet_crypto_alg_t integ_calg; - - ip46_address_t tunnel_src_addr; - ip46_address_t tunnel_dst_addr; - - fib_node_index_t fib_entry_index; - u32 sibling; - - u32 tx_fib_index; - /* Salt used in GCM modes - stored in network byte order */ u32 salt; - u64 gcm_iv_counter; + ipsec_protocol_t protocol; + u8 __pad[3]; + + /* data accessed by dataplane code should be above this comment */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline2); + + /* Elements with u64 size multiples */ union { struct @@ -205,9 +190,32 @@ typedef struct }; u64 data; } async_op_data; + + ip46_address_t tunnel_src_addr; + ip46_address_t tunnel_dst_addr; + + fib_node_t node; + + /* elements with u32 size */ + u32 id; + u32 stat_index; + vnet_crypto_alg_t integ_calg; + vnet_crypto_alg_t crypto_calg; + + fib_node_index_t fib_entry_index; + u32 sibling; + u32 tx_fib_index; + + /* else u8 packed */ + ipsec_crypto_alg_t crypto_alg; + ipsec_integ_alg_t integ_alg; + + ipsec_key_t integ_key; + ipsec_key_t crypto_key; } ipsec_sa_t; STATIC_ASSERT_OFFSET_OF (ipsec_sa_t, cacheline1, CLIB_CACHE_LINE_BYTES); +STATIC_ASSERT_OFFSET_OF (ipsec_sa_t, cacheline2, 2 * CLIB_CACHE_LINE_BYTES); #define _(a,v,s) \ always_inline int \ |