summaryrefslogtreecommitdiffstats
path: root/src/plugins/wireguard/wireguard_output_tun.c
diff options
context:
space:
mode:
authorGabriel Oginski <gabrielx.oginski@intel.com>2021-11-04 07:23:08 +0000
committerMatthew Smith <mgsmith@netgate.com>2022-01-20 14:32:27 +0000
commit8ca08496a43e8d98fe2d4130d760c6fb600d0a93 (patch)
tree761328c3f8b6db1a41bdf72acba96bbb68a01dfd /src/plugins/wireguard/wireguard_output_tun.c
parent0e4e5a8222089d6e8f70ad74afcfd4e2854a00da (diff)
wireguard: add burst mode
Originally wireguard does packet by packet encryption and decryption. This patch adds burst mode for encryption and decryption packets. In addition, it contains some performance improvement such as prefetching packet header and reducing the number of current time function calls. Type: improvement Signed-off-by: Gabriel Oginski <gabrielx.oginski@intel.com> Change-Id: I04c7daa9b6dc56cd15c789661a64ec642b35aa3f
Diffstat (limited to 'src/plugins/wireguard/wireguard_output_tun.c')
-rw-r--r--src/plugins/wireguard/wireguard_output_tun.c100
1 files changed, 77 insertions, 23 deletions
diff --git a/src/plugins/wireguard/wireguard_output_tun.c b/src/plugins/wireguard/wireguard_output_tun.c
index c792d4b713e..2feb0570a31 100644
--- a/src/plugins/wireguard/wireguard_output_tun.c
+++ b/src/plugins/wireguard/wireguard_output_tun.c
@@ -93,32 +93,67 @@ format_wg_output_tun_trace (u8 * s, va_list * args)
return s;
}
+static_always_inline void
+wg_output_process_ops (vlib_main_t *vm, vlib_node_runtime_t *node,
+ vnet_crypto_op_t *ops, vlib_buffer_t *b[], u16 *nexts,
+ u16 drop_next)
+{
+ u32 n_fail, n_ops = vec_len (ops);
+ vnet_crypto_op_t *op = ops;
+
+ if (n_ops == 0)
+ return;
+
+ n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops);
+
+ while (n_fail)
+ {
+ ASSERT (op - ops < n_ops);
+
+ if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
+ {
+ u32 bi = op->user_data;
+ b[bi]->error = node->errors[WG_OUTPUT_ERROR_KEYPAIR];
+ nexts[bi] = drop_next;
+ n_fail--;
+ }
+ op++;
+ }
+}
+
/* is_ip4 - inner header flag */
always_inline uword
wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
vlib_frame_t *frame, u8 is_ip4)
{
- u32 n_left_from;
- u32 *from;
+ wg_main_t *wmp = &wg_main;
+ wg_per_thread_data_t *ptd =
+ vec_elt_at_index (wmp->per_thread_data, vm->thread_index);
+ u32 *from = vlib_frame_vector_args (frame);
+ u32 n_left_from = frame->n_vectors;
ip4_udp_wg_header_t *hdr4_out = NULL;
ip6_udp_wg_header_t *hdr6_out = NULL;
message_data_t *message_data_wg = NULL;
- vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
- u16 nexts[VLIB_FRAME_SIZE], *next;
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
+ vnet_crypto_op_t **crypto_ops = &ptd->crypto_ops;
+ u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
+ vlib_buffer_t *sync_bufs[VLIB_FRAME_SIZE];
u32 thread_index = vm->thread_index;
-
- from = vlib_frame_vector_args (frame);
- n_left_from = frame->n_vectors;
- b = bufs;
- next = nexts;
+ u16 n_sync = 0;
+ u16 drop_next = WG_OUTPUT_NEXT_ERROR;
vlib_get_buffers (vm, from, bufs, n_left_from);
+ vec_reset_length (ptd->crypto_ops);
wg_peer_t *peer = NULL;
+ u32 adj_index = 0;
+ u32 last_adj_index = ~0;
+ index_t peeri = INDEX_INVALID;
+
+ f64 time = clib_time_now (&vm->clib_time) + vm->time_offset;
while (n_left_from > 0)
{
- index_t peeri;
u8 iph_offset = 0;
u8 is_ip4_out = 1;
u8 *plain_data;
@@ -130,14 +165,21 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
vlib_prefetch_buffer_header (b[2], LOAD);
p = vlib_buffer_get_current (b[1]);
CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (vlib_buffer_get_tail (b[1]), CLIB_CACHE_LINE_BYTES,
+ LOAD);
}
next[0] = WG_OUTPUT_NEXT_ERROR;
- peeri =
- wg_peer_get_by_adj_index (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]);
- peer = wg_peer_get (peeri);
- if (wg_peer_is_dead (peer))
+ adj_index = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
+
+ if (PREDICT_FALSE (last_adj_index != adj_index))
+ {
+ peeri = wg_peer_get_by_adj_index (adj_index);
+ peer = wg_peer_get (peeri);
+ }
+
+ if (!peer || wg_peer_is_dead (peer))
{
b[0]->error = node->errors[WG_OUTPUT_ERROR_PEER];
goto out;
@@ -179,6 +221,7 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
iph_offset = vnet_buffer (b[0])->ip.save_rewrite_length;
plain_data = vlib_buffer_get_current (b[0]) + iph_offset;
plain_data_len = vlib_buffer_length_in_chain (vm, b[0]) - iph_offset;
+ u8 *iv_data = b[0]->pre_data;
size_t encrypted_packet_len = message_data_len (plain_data_len);
@@ -194,11 +237,20 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
goto out;
}
+ if (PREDICT_FALSE (last_adj_index != adj_index))
+ {
+ wg_timers_any_authenticated_packet_sent_opt (peer, time);
+ wg_timers_data_sent_opt (peer, time);
+ wg_timers_any_authenticated_packet_traversal (peer);
+ last_adj_index = adj_index;
+ }
+
enum noise_state_crypt state;
- state = noise_remote_encrypt (
- vm, &peer->remote, &message_data_wg->receiver_index,
- &message_data_wg->counter, plain_data, plain_data_len, plain_data);
+ state = noise_sync_remote_encrypt (
+ vm, crypto_ops, &peer->remote, &message_data_wg->receiver_index,
+ &message_data_wg->counter, plain_data, plain_data_len, plain_data,
+ n_sync, iv_data, time);
if (PREDICT_FALSE (state == SC_KEEP_KEY_FRESH))
{
@@ -206,7 +258,7 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
}
else if (PREDICT_FALSE (state == SC_FAILED))
{
- //TODO: Maybe wrong
+ // TODO: Maybe wrong
wg_send_handshake_from_mt (peeri, false);
wg_peer_update_flags (peeri, WG_PEER_ESTABLISHED, false);
goto out;
@@ -236,10 +288,6 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
clib_host_to_net_u16 (b[0]->current_length);
}
- wg_timers_any_authenticated_packet_sent (peer);
- wg_timers_data_sent (peer);
- wg_timers_any_authenticated_packet_traversal (peer);
-
out:
if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
&& (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
@@ -256,12 +304,18 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
}
next:
+ sync_bufs[n_sync] = b[0];
+ n_sync += 1;
n_left_from -= 1;
next += 1;
b += 1;
}
- vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
+ /* wg-output-process-ops */
+ wg_output_process_ops (vm, node, ptd->crypto_ops, sync_bufs, nexts,
+ drop_next);
+
+ vlib_buffer_enqueue_to_next (vm, node, from, nexts, n_sync);
return frame->n_vectors;
}