aboutsummaryrefslogtreecommitdiffstats
path: root/vnet
diff options
context:
space:
mode:
Diffstat (limited to 'vnet')
-rw-r--r--vnet/vnet/ipsec/esp.h36
-rw-r--r--vnet/vnet/ipsec/esp_decrypt.c19
-rw-r--r--vnet/vnet/ipsec/esp_encrypt.c19
-rw-r--r--vnet/vnet/ipsec/ipsec.c3
-rw-r--r--vnet/vnet/ipsec/ipsec.h14
5 files changed, 58 insertions, 33 deletions
diff --git a/vnet/vnet/ipsec/esp.h b/vnet/vnet/ipsec/esp.h
index b44c26e1fa4..2334dc48bc5 100644
--- a/vnet/vnet/ipsec/esp.h
+++ b/vnet/vnet/ipsec/esp.h
@@ -50,16 +50,22 @@ typedef struct {
u8 trunc_size;
} esp_integ_alg_t;
-
typedef struct {
- esp_crypto_alg_t * esp_crypto_algs;
- esp_integ_alg_t * esp_integ_algs;
+ CLIB_CACHE_LINE_ALIGN_MARK(cacheline0);
EVP_CIPHER_CTX encrypt_ctx;
+ CLIB_CACHE_LINE_ALIGN_MARK(cacheline1);
EVP_CIPHER_CTX decrypt_ctx;
+ CLIB_CACHE_LINE_ALIGN_MARK(cacheline2);
HMAC_CTX hmac_ctx;
- ipsec_crypto_alg_t last_encrytp_alg;
- ipsec_crypto_alg_t last_decrytp_alg;
+ ipsec_crypto_alg_t last_encrypt_alg;
+ ipsec_crypto_alg_t last_decrypt_alg;
ipsec_integ_alg_t last_integ_alg;
+} esp_main_per_thread_data_t;
+
+typedef struct {
+ esp_crypto_alg_t * esp_crypto_algs;
+ esp_integ_alg_t * esp_integ_algs;
+ esp_main_per_thread_data_t * per_thread_data;
} esp_main_t;
esp_main_t esp_main;
@@ -68,6 +74,7 @@ always_inline void
esp_init()
{
esp_main_t * em = &esp_main;
+ vlib_thread_main_t * tm = vlib_get_thread_main();
memset (em, 0, sizeof (em[0]));
@@ -99,9 +106,15 @@ esp_init()
i->md = EVP_sha512();
i->trunc_size = 32;
- EVP_CIPHER_CTX_init(&(em->encrypt_ctx));
- EVP_CIPHER_CTX_init(&(em->decrypt_ctx));
- HMAC_CTX_init(&(em->hmac_ctx));
+ vec_validate_aligned(em->per_thread_data, tm->n_vlib_mains-1, CLIB_CACHE_LINE_BYTES);
+ int thread_id;
+
+ for (thread_id = 0; thread_id < tm->n_vlib_mains - 1; thread_id++)
+ {
+ EVP_CIPHER_CTX_init(&(em->per_thread_data[thread_id].encrypt_ctx));
+ EVP_CIPHER_CTX_init(&(em->per_thread_data[thread_id].decrypt_ctx));
+ HMAC_CTX_init(&(em->per_thread_data[thread_id].hmac_ctx));
+ }
}
always_inline unsigned int
@@ -115,7 +128,8 @@ hmac_calc(ipsec_integ_alg_t alg,
u32 seq_hi)
{
esp_main_t * em = &esp_main;
- HMAC_CTX * ctx = &(em->hmac_ctx);
+ u32 cpu_index = os_get_cpu_number();
+ HMAC_CTX * ctx = &(em->per_thread_data[cpu_index].hmac_ctx);
const EVP_MD * md = NULL;
unsigned int len;
@@ -124,9 +138,9 @@ hmac_calc(ipsec_integ_alg_t alg,
if (PREDICT_FALSE(em->esp_integ_algs[alg].md == 0))
return 0;
- if (PREDICT_FALSE(alg != em->last_integ_alg)) {
+ if (PREDICT_FALSE(alg != em->per_thread_data[cpu_index].last_integ_alg)) {
md = em->esp_integ_algs[alg].md;
- em->last_integ_alg = alg;
+ em->per_thread_data[cpu_index].last_integ_alg = alg;
}
HMAC_Init(ctx, key, key_len, md);
diff --git a/vnet/vnet/ipsec/esp_decrypt.c b/vnet/vnet/ipsec/esp_decrypt.c
index ad511b0fba3..958a4d6738a 100644
--- a/vnet/vnet/ipsec/esp_decrypt.c
+++ b/vnet/vnet/ipsec/esp_decrypt.c
@@ -85,7 +85,8 @@ esp_decrypt_aes_cbc(ipsec_crypto_alg_t alg,
u8 * iv)
{
esp_main_t * em = &esp_main;
- EVP_CIPHER_CTX * ctx = &(em->decrypt_ctx);
+ u32 cpu_index = os_get_cpu_number();
+ EVP_CIPHER_CTX * ctx = &(em->per_thread_data[cpu_index].decrypt_ctx);
const EVP_CIPHER * cipher = NULL;
int out_len;
@@ -94,9 +95,9 @@ esp_decrypt_aes_cbc(ipsec_crypto_alg_t alg,
if (PREDICT_FALSE(em->esp_crypto_algs[alg].type == 0))
return;
- if (PREDICT_FALSE(alg != em->last_decrytp_alg)) {
+ if (PREDICT_FALSE(alg != em->per_thread_data[cpu_index].last_decrypt_alg)) {
cipher = em->esp_crypto_algs[alg].type;
- em->last_decrytp_alg = alg;
+ em->per_thread_data[cpu_index].last_decrypt_alg = alg;
}
EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv);
@@ -235,10 +236,12 @@ esp_decrypt_node_fn (vlib_main_t * vm,
u32 * recycle = 0;
from = vlib_frame_vector_args (from_frame);
n_left_from = from_frame->n_vectors;
+ u32 cpu_index = os_get_cpu_number();
+ u32 * empty_buffers = im->empty_buffers[cpu_index];
ipsec_alloc_empty_buffers(vm, im);
- if (PREDICT_FALSE(vec_len (im->empty_buffers) < n_left_from)){
+ if (PREDICT_FALSE(vec_len (empty_buffers) < n_left_from)){
vlib_node_increment_counter (vm, esp_decrypt_node.index,
ESP_DECRYPT_ERROR_NO_BUFFER, n_left_from);
goto free_buffers_and_exit;
@@ -327,11 +330,11 @@ esp_decrypt_node_fn (vlib_main_t * vm,
}
/* grab free buffer */
- uword last_empty_buffer = vec_len (im->empty_buffers) - 1;
- o_bi0 = im->empty_buffers[last_empty_buffer];
+ uword last_empty_buffer = vec_len (empty_buffers) - 1;
+ o_bi0 = empty_buffers[last_empty_buffer];
o_b0 = vlib_get_buffer (vm, o_bi0);
- vlib_prefetch_buffer_with_index (vm, im->empty_buffers[last_empty_buffer-1], STORE);
- _vec_len (im->empty_buffers) = last_empty_buffer;
+ vlib_prefetch_buffer_with_index (vm, empty_buffers[last_empty_buffer-1], STORE);
+ _vec_len (empty_buffers) = last_empty_buffer;
/* add old buffer to the recycle list */
vec_add1(recycle, i_bi0);
diff --git a/vnet/vnet/ipsec/esp_encrypt.c b/vnet/vnet/ipsec/esp_encrypt.c
index 7194399f803..39bbf2e47ff 100644
--- a/vnet/vnet/ipsec/esp_encrypt.c
+++ b/vnet/vnet/ipsec/esp_encrypt.c
@@ -89,7 +89,8 @@ esp_encrypt_aes_cbc(ipsec_crypto_alg_t alg,
u8 * iv)
{
esp_main_t * em = &esp_main;
- EVP_CIPHER_CTX * ctx = &(em->encrypt_ctx);
+ u32 cpu_index = os_get_cpu_number();
+ EVP_CIPHER_CTX * ctx = &(em->per_thread_data[cpu_index].encrypt_ctx);
const EVP_CIPHER * cipher = NULL;
int out_len;
@@ -98,9 +99,9 @@ esp_encrypt_aes_cbc(ipsec_crypto_alg_t alg,
if (PREDICT_FALSE(em->esp_crypto_algs[alg].type == IPSEC_CRYPTO_ALG_NONE))
return;
- if (PREDICT_FALSE(alg != em->last_encrytp_alg)) {
+ if (PREDICT_FALSE(alg != em->per_thread_data[cpu_index].last_encrypt_alg)) {
cipher = em->esp_crypto_algs[alg].type;
- em->last_encrytp_alg = alg;
+ em->per_thread_data[cpu_index].last_encrypt_alg = alg;
}
EVP_EncryptInit_ex(ctx, cipher, NULL, key, iv);
@@ -142,10 +143,12 @@ esp_encrypt_node_fn (vlib_main_t * vm,
n_left_from = from_frame->n_vectors;
ipsec_main_t *im = &ipsec_main;
u32 * recycle = 0;
+ u32 cpu_index = os_get_cpu_number();
+ u32 * empty_buffers = im->empty_buffers[cpu_index];
ipsec_alloc_empty_buffers(vm, im);
- if (PREDICT_FALSE(vec_len (im->empty_buffers) < n_left_from)){
+ if (PREDICT_FALSE(vec_len (empty_buffers) < n_left_from)){
vlib_node_increment_counter (vm, esp_encrypt_node.index,
ESP_ENCRYPT_ERROR_NO_BUFFER, n_left_from);
clib_warning("no enough empty buffers. discarding frame");
@@ -197,13 +200,13 @@ esp_encrypt_node_fn (vlib_main_t * vm,
}
/* grab free buffer */
- last_empty_buffer = vec_len (im->empty_buffers) - 1;
- o_bi0 = im->empty_buffers[last_empty_buffer];
+ last_empty_buffer = vec_len (empty_buffers) - 1;
+ o_bi0 = empty_buffers[last_empty_buffer];
o_b0 = vlib_get_buffer (vm, o_bi0);
o_b0->current_data = sizeof(ethernet_header_t);
ih0 = vlib_buffer_get_current (i_b0);
- vlib_prefetch_buffer_with_index (vm, im->empty_buffers[last_empty_buffer-1], STORE);
- _vec_len (im->empty_buffers) = last_empty_buffer;
+ vlib_prefetch_buffer_with_index (vm, empty_buffers[last_empty_buffer-1], STORE);
+ _vec_len (empty_buffers) = last_empty_buffer;
to_next[0] = o_bi0;
to_next += 1;
diff --git a/vnet/vnet/ipsec/ipsec.c b/vnet/vnet/ipsec/ipsec.c
index 47beafc32b5..ea077d0a127 100644
--- a/vnet/vnet/ipsec/ipsec.c
+++ b/vnet/vnet/ipsec/ipsec.c
@@ -492,6 +492,7 @@ ipsec_init (vlib_main_t * vm)
{
clib_error_t * error;
ipsec_main_t * im = &ipsec_main;
+ vlib_thread_main_t * tm = vlib_get_thread_main();
vlib_node_t * node;
ipsec_rand_seed();
@@ -505,6 +506,8 @@ ipsec_init (vlib_main_t * vm)
im->sa_index_by_sa_id = hash_create (0, sizeof (uword));
im->spd_index_by_sw_if_index = hash_create (0, sizeof (uword));
+ vec_validate_aligned(im->empty_buffers, tm->n_vlib_mains-1, CLIB_CACHE_LINE_BYTES);
+
node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
ASSERT(node);
im->error_drop_node_index = node->index;
diff --git a/vnet/vnet/ipsec/ipsec.h b/vnet/vnet/ipsec/ipsec.h
index 386c0adecd6..8cd2b41c92f 100644
--- a/vnet/vnet/ipsec/ipsec.h
+++ b/vnet/vnet/ipsec/ipsec.h
@@ -175,7 +175,7 @@ typedef struct {
ipsec_tunnel_if_t * tunnel_interfaces;
u32 * free_tunnel_if_indices;
- u32 * empty_buffers;
+ u32 ** empty_buffers;
uword * tunnel_index_by_key;
@@ -242,20 +242,22 @@ ipsec_alloc_empty_buffers(vlib_main_t * vm, ipsec_main_t *im)
#else
u32 free_list_index = VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX;
#endif
- uword l = vec_len (im->empty_buffers);
+ u32 cpu_index = os_get_cpu_number();
+ uword l = vec_len (im->empty_buffers[cpu_index]);
uword n_alloc = 0;
if (PREDICT_FALSE(l < VLIB_FRAME_SIZE))
{
- if (!im->empty_buffers) {
- vec_alloc (im->empty_buffers, 2 * VLIB_FRAME_SIZE );
+ if (!im->empty_buffers[cpu_index]) {
+ vec_alloc (im->empty_buffers[cpu_index], 2 * VLIB_FRAME_SIZE );
}
- n_alloc = vlib_buffer_alloc_from_free_list (vm, im->empty_buffers + l,
+ n_alloc = vlib_buffer_alloc_from_free_list (vm,
+ im->empty_buffers[cpu_index] + l,
2 * VLIB_FRAME_SIZE - l,
free_list_index);
- _vec_len (im->empty_buffers) = l + n_alloc;
+ _vec_len (im->empty_buffers[cpu_index]) = l + n_alloc;
}
}