diff options
Diffstat (limited to 'vnet/vnet/ipsec')
-rw-r--r-- | vnet/vnet/ipsec/esp.h | 36 | ||||
-rw-r--r-- | vnet/vnet/ipsec/esp_decrypt.c | 19 | ||||
-rw-r--r-- | vnet/vnet/ipsec/esp_encrypt.c | 19 | ||||
-rw-r--r-- | vnet/vnet/ipsec/ipsec.c | 3 | ||||
-rw-r--r-- | vnet/vnet/ipsec/ipsec.h | 14 |
5 files changed, 58 insertions, 33 deletions
diff --git a/vnet/vnet/ipsec/esp.h b/vnet/vnet/ipsec/esp.h index b44c26e1fa4..2334dc48bc5 100644 --- a/vnet/vnet/ipsec/esp.h +++ b/vnet/vnet/ipsec/esp.h @@ -50,16 +50,22 @@ typedef struct { u8 trunc_size; } esp_integ_alg_t; - typedef struct { - esp_crypto_alg_t * esp_crypto_algs; - esp_integ_alg_t * esp_integ_algs; + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); EVP_CIPHER_CTX encrypt_ctx; + CLIB_CACHE_LINE_ALIGN_MARK(cacheline1); EVP_CIPHER_CTX decrypt_ctx; + CLIB_CACHE_LINE_ALIGN_MARK(cacheline2); HMAC_CTX hmac_ctx; - ipsec_crypto_alg_t last_encrytp_alg; - ipsec_crypto_alg_t last_decrytp_alg; + ipsec_crypto_alg_t last_encrypt_alg; + ipsec_crypto_alg_t last_decrypt_alg; ipsec_integ_alg_t last_integ_alg; +} esp_main_per_thread_data_t; + +typedef struct { + esp_crypto_alg_t * esp_crypto_algs; + esp_integ_alg_t * esp_integ_algs; + esp_main_per_thread_data_t * per_thread_data; } esp_main_t; esp_main_t esp_main; @@ -68,6 +74,7 @@ always_inline void esp_init() { esp_main_t * em = &esp_main; + vlib_thread_main_t * tm = vlib_get_thread_main(); memset (em, 0, sizeof (em[0])); @@ -99,9 +106,15 @@ esp_init() i->md = EVP_sha512(); i->trunc_size = 32; - EVP_CIPHER_CTX_init(&(em->encrypt_ctx)); - EVP_CIPHER_CTX_init(&(em->decrypt_ctx)); - HMAC_CTX_init(&(em->hmac_ctx)); + vec_validate_aligned(em->per_thread_data, tm->n_vlib_mains-1, CLIB_CACHE_LINE_BYTES); + int thread_id; + + for (thread_id = 0; thread_id < tm->n_vlib_mains - 1; thread_id++) + { + EVP_CIPHER_CTX_init(&(em->per_thread_data[thread_id].encrypt_ctx)); + EVP_CIPHER_CTX_init(&(em->per_thread_data[thread_id].decrypt_ctx)); + HMAC_CTX_init(&(em->per_thread_data[thread_id].hmac_ctx)); + } } always_inline unsigned int @@ -115,7 +128,8 @@ hmac_calc(ipsec_integ_alg_t alg, u32 seq_hi) { esp_main_t * em = &esp_main; - HMAC_CTX * ctx = &(em->hmac_ctx); + u32 cpu_index = os_get_cpu_number(); + HMAC_CTX * ctx = &(em->per_thread_data[cpu_index].hmac_ctx); const EVP_MD * md = NULL; unsigned int len; @@ -124,9 +138,9 @@ hmac_calc(ipsec_integ_alg_t alg, if (PREDICT_FALSE(em->esp_integ_algs[alg].md == 0)) return 0; - if (PREDICT_FALSE(alg != em->last_integ_alg)) { + if (PREDICT_FALSE(alg != em->per_thread_data[cpu_index].last_integ_alg)) { md = em->esp_integ_algs[alg].md; - em->last_integ_alg = alg; + em->per_thread_data[cpu_index].last_integ_alg = alg; } HMAC_Init(ctx, key, key_len, md); diff --git a/vnet/vnet/ipsec/esp_decrypt.c b/vnet/vnet/ipsec/esp_decrypt.c index ad511b0fba3..958a4d6738a 100644 --- a/vnet/vnet/ipsec/esp_decrypt.c +++ b/vnet/vnet/ipsec/esp_decrypt.c @@ -85,7 +85,8 @@ esp_decrypt_aes_cbc(ipsec_crypto_alg_t alg, u8 * iv) { esp_main_t * em = &esp_main; - EVP_CIPHER_CTX * ctx = &(em->decrypt_ctx); + u32 cpu_index = os_get_cpu_number(); + EVP_CIPHER_CTX * ctx = &(em->per_thread_data[cpu_index].decrypt_ctx); const EVP_CIPHER * cipher = NULL; int out_len; @@ -94,9 +95,9 @@ esp_decrypt_aes_cbc(ipsec_crypto_alg_t alg, if (PREDICT_FALSE(em->esp_crypto_algs[alg].type == 0)) return; - if (PREDICT_FALSE(alg != em->last_decrytp_alg)) { + if (PREDICT_FALSE(alg != em->per_thread_data[cpu_index].last_decrypt_alg)) { cipher = em->esp_crypto_algs[alg].type; - em->last_decrytp_alg = alg; + em->per_thread_data[cpu_index].last_decrypt_alg = alg; } EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv); @@ -235,10 +236,12 @@ esp_decrypt_node_fn (vlib_main_t * vm, u32 * recycle = 0; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; + u32 cpu_index = os_get_cpu_number(); + u32 * empty_buffers = im->empty_buffers[cpu_index]; ipsec_alloc_empty_buffers(vm, im); - if (PREDICT_FALSE(vec_len (im->empty_buffers) < n_left_from)){ + if (PREDICT_FALSE(vec_len (empty_buffers) < n_left_from)){ vlib_node_increment_counter (vm, esp_decrypt_node.index, ESP_DECRYPT_ERROR_NO_BUFFER, n_left_from); goto free_buffers_and_exit; @@ -327,11 +330,11 @@ esp_decrypt_node_fn (vlib_main_t * vm, } /* grab free buffer */ - uword last_empty_buffer = vec_len (im->empty_buffers) - 1; - o_bi0 = im->empty_buffers[last_empty_buffer]; + uword last_empty_buffer = vec_len (empty_buffers) - 1; + o_bi0 = empty_buffers[last_empty_buffer]; o_b0 = vlib_get_buffer (vm, o_bi0); - vlib_prefetch_buffer_with_index (vm, im->empty_buffers[last_empty_buffer-1], STORE); - _vec_len (im->empty_buffers) = last_empty_buffer; + vlib_prefetch_buffer_with_index (vm, empty_buffers[last_empty_buffer-1], STORE); + _vec_len (empty_buffers) = last_empty_buffer; /* add old buffer to the recycle list */ vec_add1(recycle, i_bi0); diff --git a/vnet/vnet/ipsec/esp_encrypt.c b/vnet/vnet/ipsec/esp_encrypt.c index 7194399f803..39bbf2e47ff 100644 --- a/vnet/vnet/ipsec/esp_encrypt.c +++ b/vnet/vnet/ipsec/esp_encrypt.c @@ -89,7 +89,8 @@ esp_encrypt_aes_cbc(ipsec_crypto_alg_t alg, u8 * iv) { esp_main_t * em = &esp_main; - EVP_CIPHER_CTX * ctx = &(em->encrypt_ctx); + u32 cpu_index = os_get_cpu_number(); + EVP_CIPHER_CTX * ctx = &(em->per_thread_data[cpu_index].encrypt_ctx); const EVP_CIPHER * cipher = NULL; int out_len; @@ -98,9 +99,9 @@ esp_encrypt_aes_cbc(ipsec_crypto_alg_t alg, if (PREDICT_FALSE(em->esp_crypto_algs[alg].type == IPSEC_CRYPTO_ALG_NONE)) return; - if (PREDICT_FALSE(alg != em->last_encrytp_alg)) { + if (PREDICT_FALSE(alg != em->per_thread_data[cpu_index].last_encrypt_alg)) { cipher = em->esp_crypto_algs[alg].type; - em->last_encrytp_alg = alg; + em->per_thread_data[cpu_index].last_encrypt_alg = alg; } EVP_EncryptInit_ex(ctx, cipher, NULL, key, iv); @@ -142,10 +143,12 @@ esp_encrypt_node_fn (vlib_main_t * vm, n_left_from = from_frame->n_vectors; ipsec_main_t *im = &ipsec_main; u32 * recycle = 0; + u32 cpu_index = os_get_cpu_number(); + u32 * empty_buffers = im->empty_buffers[cpu_index]; ipsec_alloc_empty_buffers(vm, im); - if (PREDICT_FALSE(vec_len (im->empty_buffers) < n_left_from)){ + if (PREDICT_FALSE(vec_len (empty_buffers) < n_left_from)){ vlib_node_increment_counter (vm, esp_encrypt_node.index, ESP_ENCRYPT_ERROR_NO_BUFFER, n_left_from); clib_warning("no enough empty buffers. discarding frame"); @@ -197,13 +200,13 @@ esp_encrypt_node_fn (vlib_main_t * vm, } /* grab free buffer */ - last_empty_buffer = vec_len (im->empty_buffers) - 1; - o_bi0 = im->empty_buffers[last_empty_buffer]; + last_empty_buffer = vec_len (empty_buffers) - 1; + o_bi0 = empty_buffers[last_empty_buffer]; o_b0 = vlib_get_buffer (vm, o_bi0); o_b0->current_data = sizeof(ethernet_header_t); ih0 = vlib_buffer_get_current (i_b0); - vlib_prefetch_buffer_with_index (vm, im->empty_buffers[last_empty_buffer-1], STORE); - _vec_len (im->empty_buffers) = last_empty_buffer; + vlib_prefetch_buffer_with_index (vm, empty_buffers[last_empty_buffer-1], STORE); + _vec_len (empty_buffers) = last_empty_buffer; to_next[0] = o_bi0; to_next += 1; diff --git a/vnet/vnet/ipsec/ipsec.c b/vnet/vnet/ipsec/ipsec.c index 47beafc32b5..ea077d0a127 100644 --- a/vnet/vnet/ipsec/ipsec.c +++ b/vnet/vnet/ipsec/ipsec.c @@ -492,6 +492,7 @@ ipsec_init (vlib_main_t * vm) { clib_error_t * error; ipsec_main_t * im = &ipsec_main; + vlib_thread_main_t * tm = vlib_get_thread_main(); vlib_node_t * node; ipsec_rand_seed(); @@ -505,6 +506,8 @@ ipsec_init (vlib_main_t * vm) im->sa_index_by_sa_id = hash_create (0, sizeof (uword)); im->spd_index_by_sw_if_index = hash_create (0, sizeof (uword)); + vec_validate_aligned(im->empty_buffers, tm->n_vlib_mains-1, CLIB_CACHE_LINE_BYTES); + node = vlib_get_node_by_name (vm, (u8 *) "error-drop"); ASSERT(node); im->error_drop_node_index = node->index; diff --git a/vnet/vnet/ipsec/ipsec.h b/vnet/vnet/ipsec/ipsec.h index 386c0adecd6..8cd2b41c92f 100644 --- a/vnet/vnet/ipsec/ipsec.h +++ b/vnet/vnet/ipsec/ipsec.h @@ -175,7 +175,7 @@ typedef struct { ipsec_tunnel_if_t * tunnel_interfaces; u32 * free_tunnel_if_indices; - u32 * empty_buffers; + u32 ** empty_buffers; uword * tunnel_index_by_key; @@ -242,20 +242,22 @@ ipsec_alloc_empty_buffers(vlib_main_t * vm, ipsec_main_t *im) #else u32 free_list_index = VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX; #endif - uword l = vec_len (im->empty_buffers); + u32 cpu_index = os_get_cpu_number(); + uword l = vec_len (im->empty_buffers[cpu_index]); uword n_alloc = 0; if (PREDICT_FALSE(l < VLIB_FRAME_SIZE)) { - if (!im->empty_buffers) { - vec_alloc (im->empty_buffers, 2 * VLIB_FRAME_SIZE ); + if (!im->empty_buffers[cpu_index]) { + vec_alloc (im->empty_buffers[cpu_index], 2 * VLIB_FRAME_SIZE ); } - n_alloc = vlib_buffer_alloc_from_free_list (vm, im->empty_buffers + l, + n_alloc = vlib_buffer_alloc_from_free_list (vm, + im->empty_buffers[cpu_index] + l, 2 * VLIB_FRAME_SIZE - l, free_list_index); - _vec_len (im->empty_buffers) = l + n_alloc; + _vec_len (im->empty_buffers[cpu_index]) = l + n_alloc; } } |