diff options
Diffstat (limited to 'src/vnet')
-rw-r--r-- | src/vnet/ipsec/ah_decrypt.c | 44 | ||||
-rw-r--r-- | src/vnet/ipsec/esp_decrypt.c | 44 | ||||
-rw-r--r-- | src/vnet/ipsec/ipsec_sa.c | 10 | ||||
-rw-r--r-- | src/vnet/ipsec/ipsec_sa.h | 259 |
4 files changed, 118 insertions, 239 deletions
diff --git a/src/vnet/ipsec/ah_decrypt.c b/src/vnet/ipsec/ah_decrypt.c index 6b62ff7f05c..30ea4429a75 100644 --- a/src/vnet/ipsec/ah_decrypt.c +++ b/src/vnet/ipsec/ah_decrypt.c @@ -202,16 +202,8 @@ ah_decrypt_inline (vlib_main_t * vm, pd->seq = clib_host_to_net_u32 (ah0->seq_no); /* anti-replay check */ - if (PREDICT_FALSE (irt->anti_reply_huge)) - { - anti_replay_result = ipsec_sa_anti_replay_and_sn_advance ( - irt, pd->seq, ~0, false, &pd->seq_hi, true); - } - else - { - anti_replay_result = ipsec_sa_anti_replay_and_sn_advance ( - irt, pd->seq, ~0, false, &pd->seq_hi, false); - } + anti_replay_result = ipsec_sa_anti_replay_and_sn_advance ( + irt, pd->seq, ~0, false, &pd->seq_hi); if (anti_replay_result) { ah_decrypt_set_next_index (b[0], node, vm->thread_index, @@ -317,32 +309,16 @@ ah_decrypt_inline (vlib_main_t * vm, if (PREDICT_TRUE (irt->integ_icv_size)) { /* redo the anti-reply check. see esp_decrypt for details */ - if (PREDICT_FALSE (irt->anti_reply_huge)) + if (ipsec_sa_anti_replay_and_sn_advance (irt, pd->seq, pd->seq_hi, + true, NULL)) { - if (ipsec_sa_anti_replay_and_sn_advance ( - irt, pd->seq, pd->seq_hi, true, NULL, true)) - { - ah_decrypt_set_next_index ( - b[0], node, vm->thread_index, AH_DECRYPT_ERROR_REPLAY, 0, - next, AH_DECRYPT_NEXT_DROP, pd->sa_index); - goto trace; - } - n_lost = ipsec_sa_anti_replay_advance ( - irt, thread_index, pd->seq, pd->seq_hi, true); - } - else - { - if (ipsec_sa_anti_replay_and_sn_advance ( - irt, pd->seq, pd->seq_hi, true, NULL, false)) - { - ah_decrypt_set_next_index ( - b[0], node, vm->thread_index, AH_DECRYPT_ERROR_REPLAY, 0, - next, AH_DECRYPT_NEXT_DROP, pd->sa_index); - goto trace; - } - n_lost = ipsec_sa_anti_replay_advance ( - irt, thread_index, pd->seq, pd->seq_hi, false); + ah_decrypt_set_next_index (b[0], node, vm->thread_index, + AH_DECRYPT_ERROR_REPLAY, 0, next, + AH_DECRYPT_NEXT_DROP, pd->sa_index); + goto trace; } + n_lost = ipsec_sa_anti_replay_advance (irt, thread_index, pd->seq, + pd->seq_hi); vlib_prefetch_simple_counter ( &ipsec_sa_err_counters[IPSEC_SA_ERROR_LOST], thread_index, pd->sa_index); diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c index 345a60a7fdd..1f436701970 100644 --- a/src/vnet/ipsec/esp_decrypt.c +++ b/src/vnet/ipsec/esp_decrypt.c @@ -810,32 +810,16 @@ esp_decrypt_post_crypto (vlib_main_t *vm, vlib_node_runtime_t *node, * a sequence s, s+1, s+2, s+3, ... s+n and nothing will prevent any * implementation, sequential or batching, from decrypting these. */ - if (PREDICT_FALSE (irt->anti_reply_huge)) + if (ipsec_sa_anti_replay_and_sn_advance (irt, pd->seq, pd->seq_hi, true, + NULL)) { - if (ipsec_sa_anti_replay_and_sn_advance (irt, pd->seq, pd->seq_hi, true, - NULL, true)) - { - esp_decrypt_set_next_index (b, node, vm->thread_index, - ESP_DECRYPT_ERROR_REPLAY, 0, next, - ESP_DECRYPT_NEXT_DROP, pd->sa_index); - return; - } - n_lost = ipsec_sa_anti_replay_advance (irt, vm->thread_index, pd->seq, - pd->seq_hi, true); - } - else - { - if (ipsec_sa_anti_replay_and_sn_advance (irt, pd->seq, pd->seq_hi, true, - NULL, false)) - { - esp_decrypt_set_next_index (b, node, vm->thread_index, - ESP_DECRYPT_ERROR_REPLAY, 0, next, - ESP_DECRYPT_NEXT_DROP, pd->sa_index); - return; - } - n_lost = ipsec_sa_anti_replay_advance (irt, vm->thread_index, pd->seq, - pd->seq_hi, false); + esp_decrypt_set_next_index (b, node, vm->thread_index, + ESP_DECRYPT_ERROR_REPLAY, 0, next, + ESP_DECRYPT_NEXT_DROP, pd->sa_index); + return; } + n_lost = + ipsec_sa_anti_replay_advance (irt, vm->thread_index, pd->seq, pd->seq_hi); vlib_prefetch_simple_counter (&ipsec_sa_err_counters[IPSEC_SA_ERROR_LOST], vm->thread_index, pd->sa_index); @@ -1205,16 +1189,8 @@ esp_decrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, pd->current_length = b[0]->current_length; /* anti-reply check */ - if (PREDICT_FALSE (irt->anti_reply_huge)) - { - anti_replay_result = ipsec_sa_anti_replay_and_sn_advance ( - irt, pd->seq, ~0, false, &pd->seq_hi, true); - } - else - { - anti_replay_result = ipsec_sa_anti_replay_and_sn_advance ( - irt, pd->seq, ~0, false, &pd->seq_hi, false); - } + anti_replay_result = ipsec_sa_anti_replay_and_sn_advance ( + irt, pd->seq, ~0, false, &pd->seq_hi); if (anti_replay_result) { diff --git a/src/vnet/ipsec/ipsec_sa.c b/src/vnet/ipsec/ipsec_sa.c index b59175144da..ec281cd9a11 100644 --- a/src/vnet/ipsec/ipsec_sa.c +++ b/src/vnet/ipsec/ipsec_sa.c @@ -214,7 +214,6 @@ ipsec_sa_init_runtime (ipsec_sa_t *sa) if (ipsec_sa_get_inb_rt (sa)) { ipsec_sa_inb_rt_t *irt = ipsec_sa_get_inb_rt (sa); - irt->anti_reply_huge = ipsec_sa_is_set_ANTI_REPLAY_HUGE (sa); irt->use_anti_replay = ipsec_sa_is_set_USE_ANTI_REPLAY (sa); irt->use_esn = ipsec_sa_is_set_USE_ESN (sa); irt->is_tunnel = ipsec_sa_is_set_IS_TUNNEL (sa); @@ -444,12 +443,10 @@ ipsec_sa_add_and_lock (u32 id, u32 spi, ipsec_protocol_t proto, sa->flags = flags; if (ipsec_sa_is_set_USE_ANTI_REPLAY (sa) && anti_replay_window_size > 64) - { - ipsec_sa_set_ANTI_REPLAY_HUGE (sa); - anti_replay_window_size = 1 << max_log2 (anti_replay_window_size); - } + /* window size rounded up to next power of 2 */ + anti_replay_window_size = 1 << max_log2 (anti_replay_window_size); else - anti_replay_window_size = BITS (irt->replay_window[0]); + anti_replay_window_size = 64; vec_validate (im->inb_sa_runtimes, sa_index); vec_validate (im->outb_sa_runtimes, sa_index); @@ -613,7 +610,6 @@ ipsec_sa_add_and_lock (u32 id, u32 spi, ipsec_protocol_t proto, ipsec_register_udp_port (dst_port, !ipsec_sa_is_set_IS_TUNNEL_V6 (sa)); } - /* window size rounded up to next power of 2 */ for (u32 i = 0; i < anti_replay_window_size / uword_bits; i++) irt->replay_window[i] = ~0ULL; diff --git a/src/vnet/ipsec/ipsec_sa.h b/src/vnet/ipsec/ipsec_sa.h index 346e71464de..364e9edcf65 100644 --- a/src/vnet/ipsec/ipsec_sa.h +++ b/src/vnet/ipsec/ipsec_sa.h @@ -100,8 +100,7 @@ typedef struct ipsec_key_t_ _ (32, IS_PROTECT, "Protect") \ _ (64, IS_INBOUND, "inbound") \ _ (512, IS_ASYNC, "async") \ - _ (1024, NO_ALGO_NO_DROP, "no-algo-no-drop") \ - _ (4096, ANTI_REPLAY_HUGE, "anti-replay-huge") + _ (1024, NO_ALGO_NO_DROP, "no-algo-no-drop") typedef enum ipsec_sad_flags_t_ { @@ -149,7 +148,6 @@ typedef struct u16 is_null_gmac : 1; u16 use_esn : 1; u16 use_anti_replay : 1; - u16 anti_reply_huge : 1; u16 is_protect : 1; u16 is_tunnel : 1; u16 is_transport : 1; @@ -332,14 +330,10 @@ extern uword unformat_ipsec_key (unformat_input_t *input, va_list *args); always_inline u64 ipsec_sa_anti_replay_get_64b_window (const ipsec_sa_inb_rt_t *irt) { - uword *bmp = (uword *) irt->replay_window; - - if (!irt->anti_reply_huge) - return irt->replay_window[0]; - u64 w; u32 window_size = irt->anti_replay_window_size; u32 tl_win_index = irt->seq & (window_size - 1); + uword *bmp = (uword *) irt->replay_window; if (PREDICT_TRUE (tl_win_index >= 63)) return uword_bitmap_get_multiple (bmp, tl_win_index - 63, 64); @@ -353,22 +347,15 @@ ipsec_sa_anti_replay_get_64b_window (const ipsec_sa_inb_rt_t *irt) } always_inline int -ipsec_sa_anti_replay_check (const ipsec_sa_inb_rt_t *irt, u32 seq, - bool ar_huge) +ipsec_sa_anti_replay_check (const ipsec_sa_inb_rt_t *irt, u32 window_size, + u32 seq) { - u32 window_size = irt->anti_replay_window_size; - /* we assume that the packet is in the window. * if the packet falls left (sa->seq - seq >= window size), * the result is wrong */ - if (ar_huge) - return uword_bitmap_is_bit_set ((uword *) irt->replay_window, - seq & (window_size - 1)); - else - return (irt->replay_window[0] >> (window_size + seq - irt->seq - 1)) & 1; - - return 0; + return uword_bitmap_is_bit_set ((uword *) irt->replay_window, + seq & (window_size - 1)); } /* @@ -388,7 +375,7 @@ ipsec_sa_anti_replay_check (const ipsec_sa_inb_rt_t *irt, u32 seq, always_inline int ipsec_sa_anti_replay_and_sn_advance (const ipsec_sa_inb_rt_t *irt, u32 seq, u32 hi_seq_used, bool post_decrypt, - u32 *hi_seq_req, bool ar_huge) + u32 *hi_seq_req) { ASSERT ((post_decrypt == false) == (hi_seq_req != 0)); @@ -411,7 +398,7 @@ ipsec_sa_anti_replay_and_sn_advance (const ipsec_sa_inb_rt_t *irt, u32 seq, if (irt->seq >= seq + window_size) return 1; - return ipsec_sa_anti_replay_check (irt, seq, ar_huge); + return ipsec_sa_anti_replay_check (irt, window_size, seq); } if (!irt->use_anti_replay) @@ -503,7 +490,7 @@ ipsec_sa_anti_replay_and_sn_advance (const ipsec_sa_inb_rt_t *irt, u32 seq, * The received seq number is within bounds of the window * check if it's a duplicate */ - return ipsec_sa_anti_replay_check (irt, seq, ar_huge); + return ipsec_sa_anti_replay_check (irt, window_size, seq); else /* * The received sequence number is greater than the window @@ -536,7 +523,7 @@ ipsec_sa_anti_replay_and_sn_advance (const ipsec_sa_inb_rt_t *irt, u32 seq, */ if (hi_seq_req) *hi_seq_req = irt->seq_hi; - return ipsec_sa_anti_replay_check (irt, seq, ar_huge); + return ipsec_sa_anti_replay_check (irt, window_size, seq); } else { @@ -564,7 +551,7 @@ ipsec_sa_anti_replay_and_sn_advance (const ipsec_sa_inb_rt_t *irt, u32 seq, */ if (hi_seq_req) *hi_seq_req = irt->seq_hi - 1; - return ipsec_sa_anti_replay_check (irt, seq, ar_huge); + return ipsec_sa_anti_replay_check (irt, window_size, seq); } } @@ -574,121 +561,97 @@ ipsec_sa_anti_replay_and_sn_advance (const ipsec_sa_inb_rt_t *irt, u32 seq, } always_inline u32 -ipsec_sa_anti_replay_window_shift (ipsec_sa_inb_rt_t *irt, u32 inc, - bool ar_huge) +ipsec_sa_anti_replay_window_shift (ipsec_sa_inb_rt_t *irt, u32 window_size, + u32 inc) { + uword *window = irt->replay_window; + u32 window_mask = window_size - 1; u32 n_lost = 0; u32 seen = 0; - u32 window_size = irt->anti_replay_window_size; - uword *window = irt->replay_window; if (inc < window_size) { - if (ar_huge) - { - /* the number of packets we saw in this section of the window */ - u32 window_lower_bound = (irt->seq + 1) & (window_size - 1); - u32 window_next_lower_bound = - (window_lower_bound + inc) & (window_size - 1); + /* the number of packets we saw in this section of the window */ + u32 window_lower_bound = (irt->seq + 1) & window_mask; + u32 window_next_lower_bound = (window_lower_bound + inc) & window_mask; - uword i_block, i_word_start, i_word_end, full_words; - uword n_blocks = window_size >> log2_uword_bits; - uword mask; + uword i_block, i_word_start, i_word_end, full_words; + uword n_blocks = window_size >> log2_uword_bits; + uword mask; - i_block = window_lower_bound >> log2_uword_bits; + i_block = window_lower_bound >> log2_uword_bits; - i_word_start = window_lower_bound & (uword_bits - 1); - i_word_end = window_next_lower_bound & (uword_bits - 1); + i_word_start = window_lower_bound & (uword_bits - 1); + i_word_end = window_next_lower_bound & (uword_bits - 1); - /* We stay in the same word */ - if (i_word_start + inc <= uword_bits) - { - mask = pow2_mask (inc) << i_word_start; - seen += count_set_bits (window[i_block] & mask); - window[i_block] &= ~mask; - } - else + /* We stay in the same word */ + if (i_word_start + inc <= uword_bits) + { + mask = pow2_mask (inc) << i_word_start; + seen += count_set_bits (window[i_block] & mask); + window[i_block] &= ~mask; + } + else + { + full_words = + (inc + i_word_start - uword_bits - i_word_end) >> log2_uword_bits; + + /* count set bits in the first word */ + mask = (uword) ~0 << i_word_start; + seen += count_set_bits (window[i_block] & mask); + window[i_block] &= ~mask; + i_block = (i_block + 1) & (n_blocks - 1); + + /* count set bits in the next full words */ + /* even if the last word need to be fully counted, we treat it + * apart */ + while (full_words >= 8) { - full_words = (inc + i_word_start - uword_bits - i_word_end) >> - log2_uword_bits; - - /* count set bits in the first word */ - mask = (uword) ~0 << i_word_start; - seen += count_set_bits (window[i_block] & mask); - window[i_block] &= ~mask; - i_block = (i_block + 1) & (n_blocks - 1); - - /* count set bits in the next full words */ - /* even if the last word need to be fully counted, we treat it - * apart */ - while (full_words >= 8) - { - if (full_words >= 16) - { - /* prefect the next 8 blocks (64 bytes) */ - clib_prefetch_store ( - &window[(i_block + 8) & (n_blocks - 1)]); - } - - seen += count_set_bits (window[i_block]); - seen += - count_set_bits (window[(i_block + 1) & (n_blocks - 1)]); - seen += - count_set_bits (window[(i_block + 2) & (n_blocks - 1)]); - seen += - count_set_bits (window[(i_block + 3) & (n_blocks - 1)]); - seen += - count_set_bits (window[(i_block + 4) & (n_blocks - 1)]); - seen += - count_set_bits (window[(i_block + 5) & (n_blocks - 1)]); - seen += - count_set_bits (window[(i_block + 6) & (n_blocks - 1)]); - seen += - count_set_bits (window[(i_block + 7) & (n_blocks - 1)]); - window[i_block] = 0; - window[(i_block + 1) & (n_blocks - 1)] = 0; - window[(i_block + 2) & (n_blocks - 1)] = 0; - window[(i_block + 3) & (n_blocks - 1)] = 0; - window[(i_block + 4) & (n_blocks - 1)] = 0; - window[(i_block + 5) & (n_blocks - 1)] = 0; - window[(i_block + 6) & (n_blocks - 1)] = 0; - window[(i_block + 7) & (n_blocks - 1)] = 0; - - i_block = (i_block + 8) & (n_blocks - 1); - full_words -= 8; - } - while (full_words > 0) + if (full_words >= 16) { - // last word is treated after the loop - seen += count_set_bits (window[i_block]); - window[i_block] = 0; - i_block = (i_block + 1) & (n_blocks - 1); - full_words--; + /* prefect the next 8 blocks (64 bytes) */ + clib_prefetch_store ( + &window[(i_block + 8) & (n_blocks - 1)]); } - /* the last word */ - mask = pow2_mask (i_word_end); - seen += count_set_bits (window[i_block] & mask); - window[i_block] &= ~mask; + seen += count_set_bits (window[i_block]); + seen += count_set_bits (window[(i_block + 1) & (n_blocks - 1)]); + seen += count_set_bits (window[(i_block + 2) & (n_blocks - 1)]); + seen += count_set_bits (window[(i_block + 3) & (n_blocks - 1)]); + seen += count_set_bits (window[(i_block + 4) & (n_blocks - 1)]); + seen += count_set_bits (window[(i_block + 5) & (n_blocks - 1)]); + seen += count_set_bits (window[(i_block + 6) & (n_blocks - 1)]); + seen += count_set_bits (window[(i_block + 7) & (n_blocks - 1)]); + window[i_block] = 0; + window[(i_block + 1) & (n_blocks - 1)] = 0; + window[(i_block + 2) & (n_blocks - 1)] = 0; + window[(i_block + 3) & (n_blocks - 1)] = 0; + window[(i_block + 4) & (n_blocks - 1)] = 0; + window[(i_block + 5) & (n_blocks - 1)] = 0; + window[(i_block + 6) & (n_blocks - 1)] = 0; + window[(i_block + 7) & (n_blocks - 1)] = 0; + + i_block = (i_block + 8) & (n_blocks - 1); + full_words -= 8; + } + while (full_words > 0) + { + // last word is treated after the loop + seen += count_set_bits (window[i_block]); + window[i_block] = 0; + i_block = (i_block + 1) & (n_blocks - 1); + full_words--; } - uword_bitmap_set_bits_at_index ( - window, (irt->seq + inc) & (window_size - 1), 1); - } - else - { - /* - * count how many holes there are in the portion - * of the window that we will right shift of the end - * as a result of this increments - */ - u64 old = irt->replay_window[0] & pow2_mask (inc); - /* the number of packets we saw in this section of the window */ - seen = count_set_bits (old); - irt->replay_window[0] = - ((irt->replay_window[0]) >> inc) | (1ULL << (window_size - 1)); + /* the last word */ + mask = pow2_mask (i_word_end); + seen += count_set_bits (window[i_block] & mask); + window[i_block] &= ~mask; } + uword_bitmap_set_bits_at_index (window, (irt->seq + inc) & window_mask, + 1); + /* * the number we missed is the size of the window section * minus the number we saw. @@ -699,25 +662,15 @@ ipsec_sa_anti_replay_window_shift (ipsec_sa_inb_rt_t *irt, u32 inc, { u32 n_uwords = window_size / uword_bits; /* holes in the replay window are lost packets */ - if (ar_huge) - n_lost = window_size - uword_bitmap_count_set_bits (window, n_uwords); - else - n_lost = window_size - count_set_bits (irt->replay_window[0]); + n_lost = window_size - uword_bitmap_count_set_bits (window, n_uwords); /* any sequence numbers that now fall outside the window * are forever lost */ n_lost += inc - window_size; - if (PREDICT_FALSE (ar_huge)) - { - uword_bitmap_clear (window, n_uwords); - uword_bitmap_set_bits_at_index ( - window, (irt->seq + inc) & (window_size - 1), 1); - } - else - { - irt->replay_window[0] = 1ULL << (window_size - 1); - } + uword_bitmap_clear (window, n_uwords); + uword_bitmap_set_bits_at_index (window, (irt->seq + inc) & window_mask, + 1); } return n_lost; @@ -734,10 +687,11 @@ ipsec_sa_anti_replay_window_shift (ipsec_sa_inb_rt_t *irt, u32 inc, */ always_inline u64 ipsec_sa_anti_replay_advance (ipsec_sa_inb_rt_t *irt, u32 thread_index, - u32 seq, u32 hi_seq, bool ar_huge) + u32 seq, u32 hi_seq) { u64 n_lost = 0; u32 window_size = irt->anti_replay_window_size; + u64 masked_seq = seq & (window_size - 1); u32 pos; if (irt->use_esn) @@ -747,52 +701,29 @@ ipsec_sa_anti_replay_advance (ipsec_sa_inb_rt_t *irt, u32 thread_index, if (wrap == 0 && seq > irt->seq) { pos = seq - irt->seq; - n_lost = ipsec_sa_anti_replay_window_shift (irt, pos, ar_huge); + n_lost = ipsec_sa_anti_replay_window_shift (irt, window_size, pos); irt->seq = seq; } else if (wrap > 0) { pos = seq + ~irt->seq + 1; - n_lost = ipsec_sa_anti_replay_window_shift (irt, pos, ar_huge); + n_lost = ipsec_sa_anti_replay_window_shift (irt, window_size, pos); irt->seq = seq; irt->seq_hi = hi_seq; } - else if (wrap < 0) - { - pos = ~seq + irt->seq + 1; - if (ar_huge) - uword_bitmap_set_bits_at_index (irt->replay_window, - seq & (window_size - 1), 1); - else - irt->replay_window[0] |= (1ULL << (window_size - 1 - pos)); - } else - { - pos = irt->seq - seq; - if (ar_huge) - uword_bitmap_set_bits_at_index (irt->replay_window, - seq & (window_size - 1), 1); - else - irt->replay_window[0] |= (1ULL << (window_size - 1 - pos)); - } + uword_bitmap_set_bits_at_index (irt->replay_window, masked_seq, 1); } else { if (seq > irt->seq) { pos = seq - irt->seq; - n_lost = ipsec_sa_anti_replay_window_shift (irt, pos, ar_huge); + n_lost = ipsec_sa_anti_replay_window_shift (irt, window_size, pos); irt->seq = seq; } else - { - pos = irt->seq - seq; - if (ar_huge) - uword_bitmap_set_bits_at_index (irt->replay_window, - seq & (window_size - 1), 1); - else - irt->replay_window[0] |= (1ULL << (window_size - 1 - pos)); - } + uword_bitmap_set_bits_at_index (irt->replay_window, masked_seq, 1); } return n_lost; |