dpdk: deprecate ipsec backend

Type: refactor

DPDK crypto devices are now accessible via the async infra, so
there is no need for the DPDK ipsec plugin.

In addition this patch fixes the problem that cryptodev backend
not working when master core and worker cores lies in different
numa nodes.

Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
Signed-off-by: Neale Ranns <neale@graphiant.com>
Change-Id: Ie8516bea706248c7bc25abac53a9c656bb8247d9

1 files changed, 1087 insertions, 0 deletions

diff --git a/extras/deprecated/dpdk-ipsec/ipsec.c b/extras/deprecated/dpdk-ipsec/ipsec.c
new file mode 100644
index 00000000000..e260ba7dcc4
--- /dev/null
+++ b/extras/deprecated/dpdk-ipsec/ipsec.c
@@ -0,0 +1,1087 @@
+/*
+ * Copyright (c) 2017 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <vnet/vnet.h>
+#include <vnet/ip/ip.h>
+#include <vnet/api_errno.h>
+#include <vnet/ipsec/ipsec.h>
+#include <vlib/node_funcs.h>
+#include <vlib/log.h>
+
+#include <dpdk/device/dpdk.h>
+#include <dpdk/buffer.h>
+#include <dpdk/ipsec/ipsec.h>
+
+dpdk_crypto_main_t dpdk_crypto_main;
+
+#define EMPTY_STRUCT {0}
+#define NUM_CRYPTO_MBUFS 16384
+
+static void
+algos_init (u32 n_mains)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_alg_t *a;
+
+  vec_validate_aligned (dcm->cipher_algs, IPSEC_CRYPTO_N_ALG - 1, 8);
+
+  {
+#define _(v,f,str) \
+  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].name = str; \
+  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].disabled = n_mains;
+    foreach_ipsec_crypto_alg
+#undef _
+  }
+
+  /* Minimum boundary for ciphers is 4B, required by ESP */
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_NONE];
+  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+  a->alg = RTE_CRYPTO_CIPHER_NULL;
+  a->boundary = 4;		/* 1 */
+  a->key_len = 0;
+  a->iv_len = 0;
+
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
+  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
+  a->boundary = 16;
+  a->key_len = 16;
+  a->iv_len = 16;
+
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
+  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
+  a->boundary = 16;
+  a->key_len = 24;
+  a->iv_len = 16;
+
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
+  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
+  a->boundary = 16;
+  a->key_len = 32;
+  a->iv_len = 16;
+
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_128];
+  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
+  a->boundary = 4;		/* 1 */
+  a->key_len = 16;
+  a->iv_len = 8;
+
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_192];
+  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
+  a->boundary = 4;		/* 1 */
+  a->key_len = 24;
+  a->iv_len = 8;
+
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_256];
+  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
+  a->boundary = 4;		/* 1 */
+  a->key_len = 32;
+  a->iv_len = 8;
+
+#define AES_GCM_TYPE RTE_CRYPTO_SYM_XFORM_AEAD
+#define AES_GCM_ALG RTE_CRYPTO_AEAD_AES_GCM
+
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
+  a->type = AES_GCM_TYPE;
+  a->alg = AES_GCM_ALG;
+  a->boundary = 4;		/* 1 */
+  a->key_len = 16;
+  a->iv_len = 8;
+  a->trunc_size = 16;
+
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_192];
+  a->type = AES_GCM_TYPE;
+  a->alg = AES_GCM_ALG;
+  a->boundary = 4;		/* 1 */
+  a->key_len = 24;
+  a->iv_len = 8;
+  a->trunc_size = 16;
+
+  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_256];
+  a->type = AES_GCM_TYPE;
+  a->alg = AES_GCM_ALG;
+  a->boundary = 4;		/* 1 */
+  a->key_len = 32;
+  a->iv_len = 8;
+  a->trunc_size = 16;
+
+  vec_validate (dcm->auth_algs, IPSEC_INTEG_N_ALG - 1);
+
+  {
+#define _(v,f,str) \
+  dcm->auth_algs[IPSEC_INTEG_ALG_##f].name = str; \
+  dcm->auth_algs[IPSEC_INTEG_ALG_##f].disabled = n_mains;
+    foreach_ipsec_integ_alg
+#undef _
+  }
+
+  a = &dcm->auth_algs[IPSEC_INTEG_ALG_NONE];
+  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+  a->alg = RTE_CRYPTO_AUTH_NULL;
+  a->key_len = 0;
+  a->trunc_size = 0;
+
+  a = &dcm->auth_algs[IPSEC_INTEG_ALG_MD5_96];
+  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+  a->alg = RTE_CRYPTO_AUTH_MD5_HMAC;
+  a->key_len = 16;
+  a->trunc_size = 12;
+
+  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA1_96];
+  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+  a->alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
+  a->key_len = 20;
+  a->trunc_size = 12;
+
+  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_96];
+  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
+  a->key_len = 32;
+  a->trunc_size = 12;
+
+  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_128];
+  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
+  a->key_len = 32;
+  a->trunc_size = 16;
+
+  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_384_192];
+  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+  a->alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
+  a->key_len = 48;
+  a->trunc_size = 24;
+
+  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_512_256];
+  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+  a->alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
+  a->key_len = 64;
+  a->trunc_size = 32;
+}
+
+static u8
+cipher_alg_index (const crypto_alg_t * alg)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+
+  return (alg - dcm->cipher_algs);
+}
+
+static u8
+auth_alg_index (const crypto_alg_t * alg)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+
+  return (alg - dcm->auth_algs);
+}
+
+static crypto_alg_t *
+cipher_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 key_len)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_alg_t *alg;
+
+  if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
+    return NULL;
+
+  /* *INDENT-OFF* */
+  vec_foreach (alg, dcm->cipher_algs)
+    {
+      if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
+	  (alg->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
+	  (cap->sym.cipher.algo == alg->alg) &&
+	  (alg->key_len == key_len))
+	return alg;
+      if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
+	  (alg->type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
+	  (cap->sym.aead.algo == alg->alg) &&
+	  (alg->key_len == key_len))
+	return alg;
+    }
+  /* *INDENT-ON* */
+
+  return NULL;
+}
+
+static crypto_alg_t *
+auth_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_alg_t *alg;
+
+  if ((cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) ||
+      (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH))
+    return NULL;
+
+  /* *INDENT-OFF* */
+  vec_foreach (alg, dcm->auth_algs)
+    {
+      if ((cap->sym.auth.algo == alg->alg) &&
+	  (alg->trunc_size == trunc_size))
+	return alg;
+    }
+  /* *INDENT-ON* */
+
+  return NULL;
+}
+
+static void
+crypto_set_aead_xform (struct rte_crypto_sym_xform *xform,
+		       ipsec_sa_t * sa, u8 is_outbound)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_alg_t *c;
+
+  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
+
+  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_AEAD);
+
+  xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
+  xform->aead.algo = c->alg;
+  xform->aead.key.data = sa->crypto_key.data;
+  xform->aead.key.length = c->key_len;
+  xform->aead.iv.offset =
+    crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
+  xform->aead.iv.length = 12;
+  xform->aead.digest_length = c->trunc_size;
+  xform->aead.aad_length = ipsec_sa_is_set_USE_ESN (sa) ? 12 : 8;
+  xform->next = NULL;
+
+  if (is_outbound)
+    xform->aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
+  else
+    xform->aead.op = RTE_CRYPTO_AEAD_OP_DECRYPT;
+}
+
+static void
+crypto_set_cipher_xform (struct rte_crypto_sym_xform *xform,
+			 ipsec_sa_t * sa, u8 is_outbound)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_alg_t *c;
+
+  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
+
+  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_CIPHER);
+
+  xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+  xform->cipher.algo = c->alg;
+  xform->cipher.key.data = sa->crypto_key.data;
+  xform->cipher.key.length = c->key_len;
+  xform->cipher.iv.offset =
+    crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
+  xform->cipher.iv.length = c->iv_len;
+  xform->next = NULL;
+
+  if (is_outbound)
+    xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
+  else
+    xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
+}
+
+static void
+crypto_set_auth_xform (struct rte_crypto_sym_xform *xform,
+		       ipsec_sa_t * sa, u8 is_outbound)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_alg_t *a;
+
+  a = vec_elt_at_index (dcm->auth_algs, sa->integ_alg);
+
+  ASSERT (a->type == RTE_CRYPTO_SYM_XFORM_AUTH);
+
+  xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+  xform->auth.algo = a->alg;
+  xform->auth.key.data = sa->integ_key.data;
+  xform->auth.key.length = a->key_len;
+  xform->auth.digest_length = a->trunc_size;
+  xform->next = NULL;
+
+  if (is_outbound)
+    xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
+  else
+    xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
+}
+
+clib_error_t *
+create_sym_session (struct rte_cryptodev_sym_session **session,
+		    u32 sa_idx,
+		    crypto_resource_t * res,
+		    crypto_worker_main_t * cwm, u8 is_outbound)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_data_t *data;
+  ipsec_sa_t *sa;
+  struct rte_crypto_sym_xform cipher_xform = { 0 };
+  struct rte_crypto_sym_xform auth_xform = { 0 };
+  struct rte_crypto_sym_xform *xfs;
+  struct rte_cryptodev_sym_session **s;
+  clib_error_t *error = 0;
+
+  sa = ipsec_sa_get (sa_idx);
+
+  if ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
+      (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
+      (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256))
+    {
+      crypto_set_aead_xform (&cipher_xform, sa, is_outbound);
+      xfs = &cipher_xform;
+    }
+  else
+    {
+      crypto_set_cipher_xform (&cipher_xform, sa, is_outbound);
+      crypto_set_auth_xform (&auth_xform, sa, is_outbound);
+
+      if (is_outbound)
+	{
+	  cipher_xform.next = &auth_xform;
+	  xfs = &cipher_xform;
+	}
+      else
+	{
+	  auth_xform.next = &cipher_xform;
+	  xfs = &auth_xform;
+	}
+    }
+
+  data = vec_elt_at_index (dcm->data, res->numa);
+  clib_spinlock_lock_if_init (&data->lockp);
+
+  /*
+   * DPDK_VER >= 1708:
+   *   Multiple worker/threads share the session for an SA
+   *   Single session per SA, initialized for each device driver
+   */
+  s = (void *) hash_get (data->session_by_sa_index, sa_idx);
+
+  if (!s)
+    {
+      session[0] = rte_cryptodev_sym_session_create (data->session_h);
+      if (!session[0])
+	{
+	  data->session_h_failed += 1;
+	  error = clib_error_return (0, "failed to create session header");
+	  goto done;
+	}
+      hash_set (data->session_by_sa_index, sa_idx, session[0]);
+    }
+  else
+    session[0] = s[0];
+
+  struct rte_mempool **mp;
+  mp = vec_elt_at_index (data->session_drv, res->drv_id);
+  ASSERT (mp[0] != NULL);
+
+  i32 ret =
+    rte_cryptodev_sym_session_init (res->dev_id, session[0], xfs, mp[0]);
+  if (ret)
+    {
+      data->session_drv_failed[res->drv_id] += 1;
+      error = clib_error_return (0, "failed to init session for drv %u",
+				 res->drv_id);
+      goto done;
+    }
+
+  add_session_by_drv_and_sa_idx (session[0], data, res->drv_id, sa_idx);
+
+done:
+  clib_spinlock_unlock_if_init (&data->lockp);
+  return error;
+}
+
+static void __attribute__ ((unused)) clear_and_free_obj (void *obj)
+{
+  struct rte_mempool *mp = rte_mempool_from_obj (obj);
+
+  clib_memset (obj, 0, mp->elt_size);
+
+  rte_mempool_put (mp, obj);
+}
+
+/* This is from rte_cryptodev_pmd.h */
+static inline void *
+get_session_private_data (const struct rte_cryptodev_sym_session *sess,
+			  uint8_t driver_id)
+{
+#if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
+  return sess->sess_private_data[driver_id];
+#else
+  if (unlikely (sess->nb_drivers <= driver_id))
+    return 0;
+
+  return sess->sess_data[driver_id].data;
+#endif
+}
+
+/* This is from rte_cryptodev_pmd.h */
+static inline void
+set_session_private_data (struct rte_cryptodev_sym_session *sess,
+			  uint8_t driver_id, void *private_data)
+{
+#if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
+  sess->sess_private_data[driver_id] = private_data;
+#else
+  if (unlikely (sess->nb_drivers <= driver_id))
+    return;
+  sess->sess_data[driver_id].data = private_data;
+#endif
+}
+
+static clib_error_t *
+dpdk_crypto_session_disposal (crypto_session_disposal_t * v, u64 ts)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_session_disposal_t *s;
+  void *drv_session;
+  u32 drv_id;
+  i32 ret;
+
+  /* *INDENT-OFF* */
+  vec_foreach (s, v)
+    {
+      /* ordered vector by timestamp */
+      if (!(s->ts + dcm->session_timeout < ts))
+	break;
+
+      vec_foreach_index (drv_id, dcm->drv)
+	{
+	  drv_session = get_session_private_data (s->session, drv_id);
+	  if (!drv_session)
+	    continue;
+
+	  /*
+	   * Custom clear to avoid finding a dev_id for drv_id:
+	   *  ret = rte_cryptodev_sym_session_clear (dev_id, drv_session);
+	   *  ASSERT (!ret);
+	   */
+	  clear_and_free_obj (drv_session);
+
+	  set_session_private_data (s->session, drv_id, NULL);
+	}
+
+      if (rte_mempool_from_obj(s->session))
+	{
+	  ret = rte_cryptodev_sym_session_free (s->session);
+	  ASSERT (!ret);
+	}
+    }
+  /* *INDENT-ON* */
+
+  if (s < vec_end (v))
+    vec_delete (v, s - v, 0);
+  else
+    vec_reset_length (v);
+
+  return 0;
+}
+
+static clib_error_t *
+add_del_sa_session (u32 sa_index, u8 is_add)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_data_t *data;
+  struct rte_cryptodev_sym_session *s;
+  uword *val;
+  u32 drv_id;
+
+  if (is_add)
+    return 0;
+
+  /* *INDENT-OFF* */
+  vec_foreach (data, dcm->data)
+    {
+      clib_spinlock_lock_if_init (&data->lockp);
+      val = hash_get (data->session_by_sa_index, sa_index);
+      if (val)
+        {
+          s = (struct rte_cryptodev_sym_session *) val[0];
+          vec_foreach_index (drv_id, dcm->drv)
+            {
+              val = (uword*) get_session_by_drv_and_sa_idx (data, drv_id, sa_index);
+              if (val)
+                add_session_by_drv_and_sa_idx(NULL, data, drv_id, sa_index);
+            }
+
+          hash_unset (data->session_by_sa_index, sa_index);
+
+          u64 ts = unix_time_now_nsec ();
+          dpdk_crypto_session_disposal (data->session_disposal, ts);
+
+          crypto_session_disposal_t sd;
+          sd.ts = ts;
+          sd.session = s;
+
+          vec_add1 (data->session_disposal, sd);
+        }
+      clib_spinlock_unlock_if_init (&data->lockp);
+    }
+  /* *INDENT-ON* */
+
+  return 0;
+}
+
+static clib_error_t *
+dpdk_ipsec_check_support (ipsec_sa_t * sa)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+
+  if (sa->integ_alg == IPSEC_INTEG_ALG_NONE)
+    switch (sa->crypto_alg)
+      {
+      case IPSEC_CRYPTO_ALG_NONE:
+      case IPSEC_CRYPTO_ALG_AES_GCM_128:
+      case IPSEC_CRYPTO_ALG_AES_GCM_192:
+      case IPSEC_CRYPTO_ALG_AES_GCM_256:
+	break;
+      default:
+	return clib_error_return (0, "unsupported integ-alg %U crypto-alg %U",
+				  format_ipsec_integ_alg, sa->integ_alg,
+				  format_ipsec_crypto_alg, sa->crypto_alg);
+      }
+
+  /* XXX do we need the NONE check? */
+  if (sa->crypto_alg != IPSEC_CRYPTO_ALG_NONE &&
+      dcm->cipher_algs[sa->crypto_alg].disabled)
+    return clib_error_return (0, "disabled crypto-alg %U",
+			      format_ipsec_crypto_alg, sa->crypto_alg);
+
+  /* XXX do we need the NONE check? */
+  if (sa->integ_alg != IPSEC_INTEG_ALG_NONE &&
+      dcm->auth_algs[sa->integ_alg].disabled)
+    return clib_error_return (0, "disabled integ-alg %U",
+			      format_ipsec_integ_alg, sa->integ_alg);
+  return NULL;
+}
+
+static void
+crypto_parse_capabilities (crypto_dev_t * dev,
+			   const struct rte_cryptodev_capabilities *cap,
+			   u32 n_mains)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_alg_t *alg;
+  u8 len, inc;
+
+  for (; cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; cap++)
+    {
+      /* A single capability maps to multiple cipher/auth algorithms */
+      switch (cap->sym.xform_type)
+	{
+	case RTE_CRYPTO_SYM_XFORM_AEAD:
+	case RTE_CRYPTO_SYM_XFORM_CIPHER:
+	  inc = cap->sym.cipher.key_size.increment;
+	  inc = inc ? inc : 1;
+	  for (len = cap->sym.cipher.key_size.min;
+	       len <= cap->sym.cipher.key_size.max; len += inc)
+	    {
+	      alg = cipher_cap_to_alg (cap, len);
+	      if (!alg)
+		continue;
+	      dev->cipher_support[cipher_alg_index (alg)] = 1;
+	      alg->resources += vec_len (dev->free_resources);
+	      /* At least enough resources to support one algo */
+	      dcm->enabled |= (alg->resources >= n_mains);
+	    }
+	  break;
+	case RTE_CRYPTO_SYM_XFORM_AUTH:
+	  inc = cap->sym.auth.digest_size.increment;
+	  inc = inc ? inc : 1;
+	  for (len = cap->sym.auth.digest_size.min;
+	       len <= cap->sym.auth.digest_size.max; len += inc)
+	    {
+	      alg = auth_cap_to_alg (cap, len);
+	      if (!alg)
+		continue;
+	      dev->auth_support[auth_alg_index (alg)] = 1;
+	      alg->resources += vec_len (dev->free_resources);
+	      /* At least enough resources to support one algo */
+	      dcm->enabled |= (alg->resources >= n_mains);
+	    }
+	  break;
+	default:
+	  ;
+	}
+    }
+}
+
+static clib_error_t *
+crypto_dev_conf (u8 dev, u16 n_qp, u8 numa)
+{
+  struct rte_cryptodev_config dev_conf = { 0 };
+  struct rte_cryptodev_qp_conf qp_conf = { 0 };
+  i32 ret;
+  u16 qp;
+  char *error_str;
+
+  dev_conf.socket_id = numa;
+  dev_conf.nb_queue_pairs = n_qp;
+
+  error_str = "failed to configure crypto device %u";
+  ret = rte_cryptodev_configure (dev, &dev_conf);
+  if (ret < 0)
+    return clib_error_return (0, error_str, dev);
+
+  error_str = "failed to setup crypto device %u queue pair %u";
+  qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
+  for (qp = 0; qp < n_qp; qp++)
+    {
+#if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
+      ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa, NULL);
+#else
+      ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa);
+#endif
+      if (ret < 0)
+	return clib_error_return (0, error_str, dev, qp);
+    }
+
+  error_str = "failed to start crypto device %u";
+  if (rte_cryptodev_start (dev))
+    return clib_error_return (0, error_str, dev);
+
+  return 0;
+}
+
+static void
+crypto_scan_devs (u32 n_mains)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  struct rte_cryptodev *cryptodev;
+  struct rte_cryptodev_info info = { 0 };
+  crypto_dev_t *dev;
+  crypto_resource_t *res;
+  clib_error_t *error;
+  u32 i;
+  u16 max_res_idx, res_idx, j;
+  u8 drv_id;
+
+  vec_validate_init_empty (dcm->dev, rte_cryptodev_count () - 1,
+			   (crypto_dev_t) EMPTY_STRUCT);
+
+  for (i = 0; i < rte_cryptodev_count (); i++)
+    {
+      dev = vec_elt_at_index (dcm->dev, i);
+
+      cryptodev = &rte_cryptodevs[i];
+      rte_cryptodev_info_get (i, &info);
+
+      dev->id = i;
+      dev->name = cryptodev->data->name;
+      dev->numa = rte_cryptodev_socket_id (i);
+      dev->features = info.feature_flags;
+      dev->max_qp = info.max_nb_queue_pairs;
+      drv_id = info.driver_id;
+      if (drv_id >= vec_len (dcm->drv))
+	vec_validate_init_empty (dcm->drv, drv_id,
+				 (crypto_drv_t) EMPTY_STRUCT);
+      vec_elt_at_index (dcm->drv, drv_id)->name = info.driver_name;
+      dev->drv_id = drv_id;
+      vec_add1 (vec_elt_at_index (dcm->drv, drv_id)->devs, i);
+
+      if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
+	continue;
+
+      if ((error = crypto_dev_conf (i, dev->max_qp, dev->numa)))
+	{
+	  clib_error_report (error);
+	  continue;
+	}
+
+      max_res_idx = dev->max_qp - 1;
+
+      vec_validate (dev->free_resources, max_res_idx);
+
+      res_idx = vec_len (dcm->resource);
+      vec_validate_init_empty_aligned (dcm->resource, res_idx + max_res_idx,
+				       (crypto_resource_t) EMPTY_STRUCT,
+				       CLIB_CACHE_LINE_BYTES);
+
+      for (j = 0; j <= max_res_idx; j++)
+	{
+	  vec_elt (dev->free_resources, max_res_idx - j) = res_idx + j;
+	  res = &dcm->resource[res_idx + j];
+	  res->dev_id = i;
+	  res->drv_id = drv_id;
+	  res->qp_id = j;
+	  res->numa = dev->numa;
+	  res->thread_idx = (u16) ~ 0;
+	}
+
+      crypto_parse_capabilities (dev, info.capabilities, n_mains);
+    }
+}
+
+void
+crypto_auto_placement (void)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_resource_t *res;
+  crypto_worker_main_t *cwm;
+  crypto_dev_t *dev;
+  u32 thread_idx, skip_master;
+  u16 res_idx, *idx;
+  u8 used;
+  u16 i;
+
+  skip_master = vlib_num_workers () > 0;
+
+  /* *INDENT-OFF* */
+  vec_foreach (dev, dcm->dev)
+    {
+      vec_foreach_index (thread_idx, dcm->workers_main)
+	{
+	  if (vec_len (dev->free_resources) == 0)
+	    break;
+
+	  if (thread_idx < skip_master)
+	    continue;
+
+	  /* Check thread is not already using the device */
+	  vec_foreach (idx, dev->used_resources)
+	    if (dcm->resource[idx[0]].thread_idx == thread_idx)
+	      continue;
+
+	  cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
+
+	  used = 0;
+	  res_idx = vec_pop (dev->free_resources);
+
+	  /* Set device only for supported algos */
+	  for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
+	    if (dev->cipher_support[i] &&
+		cwm->cipher_resource_idx[i] == (u16) ~0)
+	      {
+		dcm->cipher_algs[i].disabled--;
+		cwm->cipher_resource_idx[i] = res_idx;
+		used = 1;
+	      }
+
+	  for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
+	    if (dev->auth_support[i] &&
+		cwm->auth_resource_idx[i] == (u16) ~0)
+	      {
+		dcm->auth_algs[i].disabled--;
+		cwm->auth_resource_idx[i] = res_idx;
+		used = 1;
+	      }
+
+	  if (!used)
+	    {
+	      vec_add1 (dev->free_resources, res_idx);
+	      continue;
+	    }
+
+	  vec_add1 (dev->used_resources, res_idx);
+
+	  res = vec_elt_at_index (dcm->resource, res_idx);
+
+	  ASSERT (res->thread_idx == (u16) ~0);
+	  res->thread_idx = thread_idx;
+
+	  /* Add device to vector of polling resources */
+	  vec_add1 (cwm->resource_idx, res_idx);
+	}
+    }
+  /* *INDENT-ON* */
+}
+
+static void
+crypto_op_init (struct rte_mempool *mempool,
+		void *_arg __attribute__ ((unused)),
+		void *_obj, unsigned i __attribute__ ((unused)))
+{
+  struct rte_crypto_op *op = _obj;
+
+  op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
+  op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
+  op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+  op->phys_addr = rte_mempool_virt2iova (_obj);
+  op->mempool = mempool;
+}
+
+static clib_error_t *
+crypto_create_crypto_op_pool (vlib_main_t * vm, u8 numa)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  dpdk_config_main_t *conf = &dpdk_config_main;
+  crypto_data_t *data;
+  u8 *pool_name;
+  u32 pool_priv_size = sizeof (struct rte_crypto_op_pool_private);
+  struct rte_crypto_op_pool_private *priv;
+  struct rte_mempool *mp;
+
+  data = vec_elt_at_index (dcm->data, numa);
+
+  /* Already allocated */
+  if (data->crypto_op)
+    return NULL;
+
+  pool_name = format (0, "crypto_pool_numa%u%c", numa, 0);
+
+  if (conf->num_crypto_mbufs == 0)
+    conf->num_crypto_mbufs = NUM_CRYPTO_MBUFS;
+
+  mp = rte_mempool_create ((char *) pool_name, conf->num_crypto_mbufs,
+			   crypto_op_len (), 512, pool_priv_size, NULL, NULL,
+			   crypto_op_init, NULL, numa, 0);
+
+  vec_free (pool_name);
+
+  if (!mp)
+    return clib_error_return (0, "failed to create crypto op mempool");
+
+  /* Initialize mempool private data */
+  priv = rte_mempool_get_priv (mp);
+  priv->priv_size = pool_priv_size;
+  priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
+
+  data->crypto_op = mp;
+
+  return NULL;
+}
+
+static clib_error_t *
+crypto_create_session_h_pool (vlib_main_t * vm, u8 numa)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_data_t *data;
+  u8 *pool_name;
+  struct rte_mempool *mp;
+  u32 elt_size;
+
+  data = vec_elt_at_index (dcm->data, numa);
+
+  if (data->session_h)
+    return NULL;
+
+  pool_name = format (0, "session_h_pool_numa%u%c", numa, 0);
+
+
+  elt_size = rte_cryptodev_sym_get_header_session_size ();
+
+#if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
+  mp = rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
+			   elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
+#else
+  /* XXX Experimental tag in DPDK 19.02 */
+  mp = rte_cryptodev_sym_session_pool_create ((char *) pool_name,
+					      DPDK_CRYPTO_NB_SESS_OBJS,
+					      elt_size, 512, 0, numa);
+#endif
+  vec_free (pool_name);
+
+  if (!mp)
+    return clib_error_return (0, "failed to create crypto session mempool");
+
+  data->session_h = mp;
+
+  return NULL;
+}
+
+static clib_error_t *
+crypto_create_session_drv_pool (vlib_main_t * vm, crypto_dev_t * dev)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_data_t *data;
+  u8 *pool_name;
+  struct rte_mempool *mp;
+  u32 elt_size;
+  u8 numa = dev->numa;
+
+  data = vec_elt_at_index (dcm->data, numa);
+
+  vec_validate (data->session_drv, dev->drv_id);
+  vec_validate (data->session_drv_failed, dev->drv_id);
+  vec_validate_aligned (data->session_by_drv_id_and_sa_index, 32,
+			CLIB_CACHE_LINE_BYTES);
+
+  if (data->session_drv[dev->drv_id])
+    return NULL;
+
+  pool_name = format (0, "session_drv%u_pool_numa%u%c", dev->drv_id, numa, 0);
+
+  elt_size = rte_cryptodev_sym_get_private_session_size (dev->id);
+  mp =
+    rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
+			elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
+
+  vec_free (pool_name);
+
+  if (!mp)
+    return clib_error_return (0, "failed to create session drv mempool");
+
+  data->session_drv[dev->drv_id] = mp;
+  clib_spinlock_init (&data->lockp);
+
+  return NULL;
+}
+
+static clib_error_t *
+crypto_create_pools (vlib_main_t * vm)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  clib_error_t *error = NULL;
+  crypto_dev_t *dev;
+
+  /* *INDENT-OFF* */
+  vec_foreach (dev, dcm->dev)
+    {
+      vec_validate_aligned (dcm->data, dev->numa, CLIB_CACHE_LINE_BYTES);
+
+      error = crypto_create_crypto_op_pool (vm, dev->numa);
+      if (error)
+	return error;
+
+      error = crypto_create_session_h_pool (vm, dev->numa);
+      if (error)
+	return error;
+
+      error = crypto_create_session_drv_pool (vm, dev);
+      if (error)
+	return error;
+    }
+  /* *INDENT-ON* */
+
+  return NULL;
+}
+
+static void
+crypto_disable (void)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_data_t *data;
+  u8 i;
+
+  dcm->enabled = 0;
+
+  /* *INDENT-OFF* */
+  vec_foreach (data, dcm->data)
+    {
+      rte_mempool_free (data->crypto_op);
+      rte_mempool_free (data->session_h);
+
+      vec_foreach_index (i, data->session_drv)
+	rte_mempool_free (data->session_drv[i]);
+
+      vec_free (data->session_drv);
+      clib_spinlock_free (&data->lockp);
+    }
+  /* *INDENT-ON* */
+
+  vec_free (dcm->data);
+  vec_free (dcm->workers_main);
+  vec_free (dcm->dev);
+  vec_free (dcm->resource);
+  vec_free (dcm->cipher_algs);
+  vec_free (dcm->auth_algs);
+}
+
+static clib_error_t *
+dpdk_ipsec_enable_disable (int is_enable)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "dpdk-crypto-input");
+  u32 skip_master = vlib_num_workers () > 0;
+  u32 n_mains = tm->n_vlib_mains;
+  u32 i;
+
+  ASSERT (node);
+  for (i = skip_master; i < n_mains; i++)
+    vlib_node_set_state (vlib_mains[i], node->index, is_enable != 0 ?
+			 VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED);
+
+  return 0;
+}
+
+static clib_error_t *
+dpdk_ipsec_main_init (vlib_main_t * vm)
+{
+  ipsec_main_t *im = &ipsec_main;
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  crypto_worker_main_t *cwm;
+  clib_error_t *error = NULL;
+  u32 skip_master, n_mains;
+
+  n_mains = tm->n_vlib_mains;
+  skip_master = vlib_num_workers () > 0;
+
+  algos_init (n_mains - skip_master);
+
+  crypto_scan_devs (n_mains - skip_master);
+
+  if (!(dcm->enabled))
+    {
+      vlib_log_warn (dpdk_main.log_default,
+		     "not enough DPDK crypto resources");
+      crypto_disable ();
+      return 0;
+    }
+
+  dcm->session_timeout = 10e9;
+
+  vec_validate_init_empty_aligned (dcm->workers_main, n_mains - 1,
+				   (crypto_worker_main_t) EMPTY_STRUCT,
+				   CLIB_CACHE_LINE_BYTES);
+
+  /* *INDENT-OFF* */
+  vec_foreach (cwm, dcm->workers_main)
+    {
+      vec_validate_init_empty_aligned (cwm->ops, VLIB_FRAME_SIZE - 1, 0,
+				       CLIB_CACHE_LINE_BYTES);
+      clib_memset (cwm->cipher_resource_idx, ~0,
+	      IPSEC_CRYPTO_N_ALG * sizeof(*cwm->cipher_resource_idx));
+      clib_memset (cwm->auth_resource_idx, ~0,
+	      IPSEC_INTEG_N_ALG * sizeof(*cwm->auth_resource_idx));
+    }
+  /* *INDENT-ON* */
+
+  crypto_auto_placement ();
+
+  error = crypto_create_pools (vm);
+  if (error)
+    {
+      clib_error_report (error);
+      crypto_disable ();
+      return 0;
+    }
+
+  u32 idx = ipsec_register_esp_backend (
+    vm, im, "dpdk backend", "dpdk-esp4-encrypt", "dpdk-esp4-encrypt-tun",
+    "dpdk-esp4-decrypt", "dpdk-esp4-decrypt", "dpdk-esp6-encrypt",
+    "dpdk-esp6-encrypt-tun", "dpdk-esp6-decrypt", "dpdk-esp6-decrypt",
+    "error-drop", dpdk_ipsec_check_support, add_del_sa_session,
+    dpdk_ipsec_enable_disable);
+  int rv;
+  if (im->esp_current_backend == ~0)
+    {
+      rv = ipsec_select_esp_backend (im, idx);
+      ASSERT (rv == 0);
+    }
+  return 0;
+}
+
+VLIB_MAIN_LOOP_ENTER_FUNCTION (dpdk_ipsec_main_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */