aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/plugins/dpdk/ipsec/crypto_node.c145
-rw-r--r--src/plugins/dpdk/ipsec/ipsec.c23
-rw-r--r--src/plugins/dpdk/ipsec/ipsec.h58
-rw-r--r--src/vnet/ipsec/ipsec_output.c12
4 files changed, 119 insertions, 119 deletions
diff --git a/src/plugins/dpdk/ipsec/crypto_node.c b/src/plugins/dpdk/ipsec/crypto_node.c
index 89c5068c0f9..6b9ff58efcf 100644
--- a/src/plugins/dpdk/ipsec/crypto_node.c
+++ b/src/plugins/dpdk/ipsec/crypto_node.c
@@ -102,74 +102,72 @@ dpdk_crypto_dequeue (vlib_main_t * vm, vlib_node_runtime_t * node,
next_index = node->cached_next_index;
- do
- {
- ops = cwm->ops;
- n_ops = rte_cryptodev_dequeue_burst (res->dev_id,
- res->qp_id + outbound,
- ops, VLIB_FRAME_SIZE);
- res->inflights[outbound] -= n_ops;
- ASSERT (res->inflights >= 0);
-
- n_deq = n_ops;
- total_n_deq += n_ops;
-
- while (n_ops > 0)
- {
- u32 n_left_to_next;
-
- vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
- while (n_ops > 0 && n_left_to_next > 0)
- {
- u32 bi0, next0;
- vlib_buffer_t *b0 = 0;
- struct rte_crypto_op *op;
-
- op = ops[0];
- ops += 1;
- n_ops -= 1;
- n_left_to_next -= 1;
-
- dpdk_op_priv_t *priv = crypto_op_get_priv (op);
- next0 = priv->next;
-
- if (PREDICT_FALSE (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
- {
- next0 = DPDK_CRYPTO_INPUT_NEXT_DROP;
- vlib_node_increment_counter (vm,
- dpdk_crypto_input_node.index,
- DPDK_CRYPTO_INPUT_ERROR_STATUS,
- 1);
- }
-
- /* XXX store bi0 and next0 in op private? */
-
- b0 = vlib_buffer_from_rte_mbuf (op->sym[0].m_src);
- bi0 = vlib_get_buffer_index (vm, b0);
-
- to_next[0] = bi0;
- to_next += 1;
-
- if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
- {
- vlib_trace_next_frame (vm, node, next0);
- dpdk_crypto_input_trace_t *tr =
- vlib_add_trace (vm, node, b0, sizeof (*tr));
- tr->status = op->status;
- }
-
- op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
-
- vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
- n_left_to_next, bi0, next0);
- }
- vlib_put_next_frame (vm, node, next_index, n_left_to_next);
- }
-
- crypto_free_ops (numa, cwm->ops, n_deq);
- }
- while (n_deq == VLIB_FRAME_SIZE && res->inflights[outbound]);
+ {
+ ops = cwm->ops;
+ n_ops = rte_cryptodev_dequeue_burst (res->dev_id,
+ res->qp_id + outbound,
+ ops, VLIB_FRAME_SIZE);
+ res->inflights[outbound] -= n_ops;
+ ASSERT (res->inflights >= 0);
+
+ n_deq = n_ops;
+ total_n_deq += n_ops;
+
+ while (n_ops > 0)
+ {
+ u32 n_left_to_next;
+
+ vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+ while (n_ops > 0 && n_left_to_next > 0)
+ {
+ u32 bi0, next0;
+ vlib_buffer_t *b0 = 0;
+ struct rte_crypto_op *op;
+
+ op = ops[0];
+ ops += 1;
+ n_ops -= 1;
+ n_left_to_next -= 1;
+
+ dpdk_op_priv_t *priv = crypto_op_get_priv (op);
+ next0 = priv->next;
+
+ if (PREDICT_FALSE (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
+ {
+ next0 = DPDK_CRYPTO_INPUT_NEXT_DROP;
+ vlib_node_increment_counter (vm,
+ dpdk_crypto_input_node.index,
+ DPDK_CRYPTO_INPUT_ERROR_STATUS,
+ 1);
+ }
+
+ /* XXX store bi0 and next0 in op private? */
+
+ b0 = vlib_buffer_from_rte_mbuf (op->sym[0].m_src);
+ bi0 = vlib_get_buffer_index (vm, b0);
+
+ to_next[0] = bi0;
+ to_next += 1;
+
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ vlib_trace_next_frame (vm, node, next0);
+ dpdk_crypto_input_trace_t *tr =
+ vlib_add_trace (vm, node, b0, sizeof (*tr));
+ tr->status = op->status;
+ }
+
+ op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+
+ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+ n_left_to_next, bi0, next0);
+ }
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ }
+
+ crypto_free_ops (numa, cwm->ops, n_deq);
+ }
vlib_node_increment_counter (vm, dpdk_crypto_input_node.index,
DPDK_CRYPTO_INPUT_ERROR_DQ_COPS, total_n_deq);
@@ -185,7 +183,6 @@ dpdk_crypto_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
crypto_worker_main_t *cwm = &dcm->workers_main[thread_index];
crypto_resource_t *res;
u32 n_deq = 0;
- u8 outbound;
u16 *remove = NULL, *res_idx;
word i;
@@ -194,13 +191,11 @@ dpdk_crypto_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
{
res = vec_elt_at_index (dcm->resource, res_idx[0]);
- outbound = 0;
- if (res->inflights[outbound])
- n_deq += dpdk_crypto_dequeue (vm, node, res, outbound);
+ if (res->inflights[0])
+ n_deq += dpdk_crypto_dequeue (vm, node, res, 0);
- outbound = 1;
- if (res->inflights[outbound])
- n_deq += dpdk_crypto_dequeue (vm, node, res, outbound);
+ if (res->inflights[1])
+ n_deq += dpdk_crypto_dequeue (vm, node, res, 1);
if (unlikely(res->remove && !(res->inflights[0] || res->inflights[1])))
vec_add1 (remove, res_idx[0]);
diff --git a/src/plugins/dpdk/ipsec/ipsec.c b/src/plugins/dpdk/ipsec/ipsec.c
index 5933cf9573e..b8cfc7e6b78 100644
--- a/src/plugins/dpdk/ipsec/ipsec.c
+++ b/src/plugins/dpdk/ipsec/ipsec.c
@@ -329,11 +329,8 @@ create_sym_session (struct rte_cryptodev_sym_session **session,
struct rte_crypto_sym_xform auth_xform = { 0 };
struct rte_crypto_sym_xform *xfs;
struct rte_cryptodev_sym_session **s;
- crypto_session_key_t key = { 0 };
clib_error_t *erorr = 0;
- key.drv_id = res->drv_id;
- key.sa_idx = sa_idx;
sa = pool_elt_at_index (im->sad, sa_idx);
@@ -399,7 +396,7 @@ create_sym_session (struct rte_cryptodev_sym_session **session,
goto done;
}
- hash_set (data->session_by_drv_id_and_sa_index, key.val, session[0]);
+ add_session_by_drv_and_sa_idx (session[0], data, res->drv_id, sa_idx);
done:
clib_spinlock_unlock_if_init (&data->lockp);
@@ -486,7 +483,6 @@ add_del_sa_session (u32 sa_index, u8 is_add)
dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
crypto_data_t *data;
struct rte_cryptodev_sym_session *s;
- crypto_session_key_t key = { 0 };
uword *val;
u32 drv_id;
@@ -510,23 +506,19 @@ add_del_sa_session (u32 sa_index, u8 is_add)
return 0;
}
- key.sa_idx = sa_index;
-
/* *INDENT-OFF* */
vec_foreach (data, dcm->data)
{
clib_spinlock_lock_if_init (&data->lockp);
val = hash_get (data->session_by_sa_index, sa_index);
- s = (struct rte_cryptodev_sym_session *) val;
- if (s)
+ if (val)
{
+ s = (struct rte_cryptodev_sym_session *) val[0];
vec_foreach_index (drv_id, dcm->drv)
{
- key.drv_id = drv_id;
- val = hash_get (data->session_by_drv_id_and_sa_index, key.val);
- s = (struct rte_cryptodev_sym_session *) val;
- if (s)
- hash_unset (data->session_by_drv_id_and_sa_index, key.val);
+ val = (uword*) get_session_by_drv_and_sa_idx (data, drv_id, sa_index);
+ if (val)
+ add_session_by_drv_and_sa_idx(NULL, data, drv_id, sa_index);
}
hash_unset (data->session_by_sa_index, sa_index);
@@ -913,6 +905,8 @@ crypto_create_session_drv_pool (vlib_main_t * vm, crypto_dev_t * dev)
vec_validate (data->session_drv, dev->drv_id);
vec_validate (data->session_drv_failed, dev->drv_id);
+ vec_validate_aligned (data->session_by_drv_id_and_sa_index, 32,
+ CLIB_CACHE_LINE_BYTES);
if (data->session_drv[dev->drv_id])
return NULL;
@@ -989,7 +983,6 @@ crypto_disable (void)
vec_free (dcm->data);
vec_free (dcm->workers_main);
- vec_free (dcm->sa_session);
vec_free (dcm->dev);
vec_free (dcm->resource);
vec_free (dcm->cipher_algs);
diff --git a/src/plugins/dpdk/ipsec/ipsec.h b/src/plugins/dpdk/ipsec/ipsec.h
index 4287a2a9372..775e7521ffe 100644
--- a/src/plugins/dpdk/ipsec/ipsec.h
+++ b/src/plugins/dpdk/ipsec/ipsec.h
@@ -127,6 +127,12 @@ typedef struct
typedef struct
{
+ struct rte_cryptodev_sym_session *session;
+ u64 dev_mask;
+} crypto_session_by_drv_t;
+
+typedef struct
+{
/* Required for vec_validate_aligned */
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
struct rte_mempool *crypto_op;
@@ -134,17 +140,16 @@ typedef struct
struct rte_mempool **session_drv;
crypto_session_disposal_t *session_disposal;
uword *session_by_sa_index;
- uword *session_by_drv_id_and_sa_index;
u64 crypto_op_get_failed;
u64 session_h_failed;
u64 *session_drv_failed;
+ crypto_session_by_drv_t *session_by_drv_id_and_sa_index;
clib_spinlock_t lockp;
} crypto_data_t;
typedef struct
{
crypto_worker_main_t *workers_main;
- struct rte_cryptodev_sym_session **sa_session;
crypto_dev_t *dev;
crypto_resource_t *resource;
crypto_alg_t *cipher_algs;
@@ -194,38 +199,47 @@ crypto_op_get_priv (struct rte_crypto_op * op)
return (dpdk_op_priv_t *) (((u8 *) op) + crypto_op_get_priv_offset ());
}
-/* XXX this requires 64 bit builds so hash_xxx macros use u64 key */
-typedef union
+
+static_always_inline void
+add_session_by_drv_and_sa_idx (struct rte_cryptodev_sym_session *session,
+ crypto_data_t * data, u32 drv_id, u32 sa_idx)
{
- u64 val;
- struct
- {
- u32 drv_id;
- u32 sa_idx;
- };
-} crypto_session_key_t;
+ crypto_session_by_drv_t *sbd;
+ vec_validate_aligned (data->session_by_drv_id_and_sa_index, sa_idx,
+ CLIB_CACHE_LINE_BYTES);
+ sbd = vec_elt_at_index (data->session_by_drv_id_and_sa_index, sa_idx);
+ sbd->dev_mask |= 1L << drv_id;
+ sbd->session = session;
+}
+
+static_always_inline struct rte_cryptodev_sym_session *
+get_session_by_drv_and_sa_idx (crypto_data_t * data, u32 drv_id, u32 sa_idx)
+{
+ crypto_session_by_drv_t *sess_by_sa;
+ if (_vec_len (data->session_by_drv_id_and_sa_index) <= sa_idx)
+ return NULL;
+ sess_by_sa =
+ vec_elt_at_index (data->session_by_drv_id_and_sa_index, sa_idx);
+ return (sess_by_sa->dev_mask & (1L << drv_id)) ? sess_by_sa->session : NULL;
+}
static_always_inline clib_error_t *
-crypto_get_session (struct rte_cryptodev_sym_session **session,
+crypto_get_session (struct rte_cryptodev_sym_session ** session,
u32 sa_idx,
crypto_resource_t * res,
crypto_worker_main_t * cwm, u8 is_outbound)
{
dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
crypto_data_t *data;
- uword *val;
- crypto_session_key_t key = { 0 };
-
- key.drv_id = res->drv_id;
- key.sa_idx = sa_idx;
+ struct rte_cryptodev_sym_session *sess;
data = vec_elt_at_index (dcm->data, res->numa);
- val = hash_get (data->session_by_drv_id_and_sa_index, key.val);
+ sess = get_session_by_drv_and_sa_idx (data, res->drv_id, sa_idx);
- if (PREDICT_FALSE (!val))
+ if (PREDICT_FALSE (!sess))
return create_sym_session (session, sa_idx, res, cwm, is_outbound);
- session[0] = (struct rte_cryptodev_sym_session *) val[0];
+ session[0] = sess;
return NULL;
}
@@ -239,8 +253,8 @@ get_resource (crypto_worker_main_t * cwm, ipsec_sa_t * sa)
/* Not allowed to setup SA with no-aead-cipher/NULL or NULL/NULL */
- is_aead = ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
- (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
+ is_aead = ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) ||
+ (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) ||
(sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256));
if (sa->crypto_alg == IPSEC_CRYPTO_ALG_NONE)
diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c
index d56b665157d..4bfbf603072 100644
--- a/src/vnet/ipsec/ipsec_output.c
+++ b/src/vnet/ipsec/ipsec_output.c
@@ -88,16 +88,16 @@ ipsec_output_policy_match (ipsec_spd_t * spd, u8 pr, u32 la, u32 ra, u16 lp,
if (PREDICT_FALSE (p->protocol && (p->protocol != pr)))
continue;
- if (la < clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
+ if (ra < clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
continue;
- if (la > clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
+ if (ra > clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
continue;
- if (ra < clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
+ if (la < clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
continue;
- if (ra > clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
+ if (la > clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
continue;
if (PREDICT_FALSE
@@ -274,11 +274,9 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
{
if (p0->policy == IPSEC_POLICY_ACTION_PROTECT)
{
- u32 sa_index = 0;
ipsec_sa_t *sa = 0;
nc_protect++;
- sa_index = ipsec_get_sa_index_by_sa_id (p0->sa_id);
- sa = pool_elt_at_index (im->sad, sa_index);
+ sa = pool_elt_at_index (im->sad, p0->sa_index);
if (sa->protocol == IPSEC_PROTOCOL_ESP)
next_node_index = im->esp_encrypt_node_index;
else