New upstream version 17.08

Change-Id: I288b50990f52646089d6b1f3aaa6ba2f091a51d7
Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>

1 files changed, 158 insertions, 116 deletions

diff --git a/app/test-crypto-perf/cperf_ops.c b/app/test-crypto-perf/cperf_ops.c
index c2c3db57..88fb9725 100644
--- a/app/test-crypto-perf/cperf_ops.c
+++ b/app/test-crypto-perf/cperf_ops.c
@@ -40,7 +40,8 @@ cperf_set_ops_null_cipher(struct rte_crypto_op **ops,
 		struct rte_mbuf **bufs_in, struct rte_mbuf **bufs_out,
 		uint16_t nb_ops, struct rte_cryptodev_sym_session *sess,
 		const struct cperf_options *options,
-		const struct cperf_test_vector *test_vector __rte_unused)
+		const struct cperf_test_vector *test_vector __rte_unused,
+		uint16_t iv_offset __rte_unused)
 {
 	uint16_t i;
 
@@ -65,7 +66,8 @@ cperf_set_ops_null_auth(struct rte_crypto_op **ops,
 		struct rte_mbuf **bufs_in, struct rte_mbuf **bufs_out,
 		uint16_t nb_ops, struct rte_cryptodev_sym_session *sess,
 		const struct cperf_options *options,
-		const struct cperf_test_vector *test_vector __rte_unused)
+		const struct cperf_test_vector *test_vector __rte_unused,
+		uint16_t iv_offset __rte_unused)
 {
 	uint16_t i;
 
@@ -90,7 +92,8 @@ cperf_set_ops_cipher(struct rte_crypto_op **ops,
 		struct rte_mbuf **bufs_in, struct rte_mbuf **bufs_out,
 		uint16_t nb_ops, struct rte_cryptodev_sym_session *sess,
 		const struct cperf_options *options,
-		const struct cperf_test_vector *test_vector)
+		const struct cperf_test_vector *test_vector,
+		uint16_t iv_offset)
 {
 	uint16_t i;
 
@@ -103,10 +106,6 @@ cperf_set_ops_cipher(struct rte_crypto_op **ops,
 		sym_op->m_dst = bufs_out[i];
 
 		/* cipher parameters */
-		sym_op->cipher.iv.data = test_vector->iv.data;
-		sym_op->cipher.iv.phys_addr = test_vector->iv.phys_addr;
-		sym_op->cipher.iv.length = test_vector->iv.length;
-
 		if (options->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
 				options->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 ||
 				options->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3)
@@ -117,6 +116,17 @@ cperf_set_ops_cipher(struct rte_crypto_op **ops,
 		sym_op->cipher.data.offset = 0;
 	}
 
+	if (options->test == CPERF_TEST_TYPE_VERIFY) {
+		for (i = 0; i < nb_ops; i++) {
+			uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i],
+					uint8_t *, iv_offset);
+
+			memcpy(iv_ptr, test_vector->cipher_iv.data,
+					test_vector->cipher_iv.length);
+
+		}
+	}
+
 	return 0;
 }
 
@@ -125,7 +135,8 @@ cperf_set_ops_auth(struct rte_crypto_op **ops,
 		struct rte_mbuf **bufs_in, struct rte_mbuf **bufs_out,
 		uint16_t nb_ops, struct rte_cryptodev_sym_session *sess,
 		const struct cperf_options *options,
-		const struct cperf_test_vector *test_vector)
+		const struct cperf_test_vector *test_vector,
+		uint16_t iv_offset)
 {
 	uint16_t i;
 
@@ -137,12 +148,19 @@ cperf_set_ops_auth(struct rte_crypto_op **ops,
 		sym_op->m_src = bufs_in[i];
 		sym_op->m_dst = bufs_out[i];
 
+		if (test_vector->auth_iv.length) {
+			uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i],
+								uint8_t *,
+								iv_offset);
+			memcpy(iv_ptr, test_vector->auth_iv.data,
+					test_vector->auth_iv.length);
+		}
+
 		/* authentication parameters */
 		if (options->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) {
 			sym_op->auth.digest.data = test_vector->digest.data;
 			sym_op->auth.digest.phys_addr =
 					test_vector->digest.phys_addr;
-			sym_op->auth.digest.length = options->auth_digest_sz;
 		} else {
 
 			uint32_t offset = options->test_buffer_size;
@@ -151,24 +169,19 @@ cperf_set_ops_auth(struct rte_crypto_op **ops,
 			if (options->out_of_place) {
 				buf =  bufs_out[i];
 			} else {
-				buf =  bufs_in[i];
-
-				tbuf = buf;
+				tbuf =  bufs_in[i];
 				while ((tbuf->next != NULL) &&
 						(offset >= tbuf->data_len)) {
 					offset -= tbuf->data_len;
 					tbuf = tbuf->next;
 				}
+				buf = tbuf;
 			}
 
 			sym_op->auth.digest.data = rte_pktmbuf_mtod_offset(buf,
 					uint8_t *, offset);
 			sym_op->auth.digest.phys_addr =
 					rte_pktmbuf_mtophys_offset(buf,	offset);
-			sym_op->auth.digest.length = options->auth_digest_sz;
-			sym_op->auth.aad.phys_addr = test_vector->aad.phys_addr;
-			sym_op->auth.aad.data = test_vector->aad.data;
-			sym_op->auth.aad.length = options->auth_aad_sz;
 
 		}
 
@@ -182,6 +195,17 @@ cperf_set_ops_auth(struct rte_crypto_op **ops,
 		sym_op->auth.data.offset = 0;
 	}
 
+	if (options->test == CPERF_TEST_TYPE_VERIFY) {
+		if (test_vector->auth_iv.length) {
+			for (i = 0; i < nb_ops; i++) {
+				uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i],
+						uint8_t *, iv_offset);
+
+				memcpy(iv_ptr, test_vector->auth_iv.data,
+						test_vector->auth_iv.length);
+			}
+		}
+	}
 	return 0;
 }
 
@@ -190,7 +214,8 @@ cperf_set_ops_cipher_auth(struct rte_crypto_op **ops,
 		struct rte_mbuf **bufs_in, struct rte_mbuf **bufs_out,
 		uint16_t nb_ops, struct rte_cryptodev_sym_session *sess,
 		const struct cperf_options *options,
-		const struct cperf_test_vector *test_vector)
+		const struct cperf_test_vector *test_vector,
+		uint16_t iv_offset)
 {
 	uint16_t i;
 
@@ -203,10 +228,6 @@ cperf_set_ops_cipher_auth(struct rte_crypto_op **ops,
 		sym_op->m_dst = bufs_out[i];
 
 		/* cipher parameters */
-		sym_op->cipher.iv.data = test_vector->iv.data;
-		sym_op->cipher.iv.phys_addr = test_vector->iv.phys_addr;
-		sym_op->cipher.iv.length = test_vector->iv.length;
-
 		if (options->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
 				options->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 ||
 				options->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3)
@@ -221,7 +242,6 @@ cperf_set_ops_cipher_auth(struct rte_crypto_op **ops,
 			sym_op->auth.digest.data = test_vector->digest.data;
 			sym_op->auth.digest.phys_addr =
 					test_vector->digest.phys_addr;
-			sym_op->auth.digest.length = options->auth_digest_sz;
 		} else {
 
 			uint32_t offset = options->test_buffer_size;
@@ -230,24 +250,19 @@ cperf_set_ops_cipher_auth(struct rte_crypto_op **ops,
 			if (options->out_of_place) {
 				buf =  bufs_out[i];
 			} else {
-				buf =  bufs_in[i];
-
-				tbuf = buf;
+				tbuf =  bufs_in[i];
 				while ((tbuf->next != NULL) &&
 						(offset >= tbuf->data_len)) {
 					offset -= tbuf->data_len;
 					tbuf = tbuf->next;
 				}
+				buf = tbuf;
 			}
 
 			sym_op->auth.digest.data = rte_pktmbuf_mtod_offset(buf,
 					uint8_t *, offset);
 			sym_op->auth.digest.phys_addr =
 					rte_pktmbuf_mtophys_offset(buf,	offset);
-			sym_op->auth.digest.length = options->auth_digest_sz;
-			sym_op->auth.aad.phys_addr = test_vector->aad.phys_addr;
-			sym_op->auth.aad.data = test_vector->aad.data;
-			sym_op->auth.aad.length = options->auth_aad_sz;
 		}
 
 		if (options->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
@@ -260,6 +275,26 @@ cperf_set_ops_cipher_auth(struct rte_crypto_op **ops,
 		sym_op->auth.data.offset = 0;
 	}
 
+	if (options->test == CPERF_TEST_TYPE_VERIFY) {
+		for (i = 0; i < nb_ops; i++) {
+			uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i],
+					uint8_t *, iv_offset);
+
+			memcpy(iv_ptr, test_vector->cipher_iv.data,
+					test_vector->cipher_iv.length);
+			if (test_vector->auth_iv.length) {
+				/*
+				 * Copy IV after the crypto operation and
+				 * the cipher IV
+				 */
+				iv_ptr += test_vector->cipher_iv.length;
+				memcpy(iv_ptr, test_vector->auth_iv.data,
+						test_vector->auth_iv.length);
+			}
+		}
+
+	}
+
 	return 0;
 }
 
@@ -268,7 +303,8 @@ cperf_set_ops_aead(struct rte_crypto_op **ops,
 		struct rte_mbuf **bufs_in, struct rte_mbuf **bufs_out,
 		uint16_t nb_ops, struct rte_cryptodev_sym_session *sess,
 		const struct cperf_options *options,
-		const struct cperf_test_vector *test_vector)
+		const struct cperf_test_vector *test_vector,
+		uint16_t iv_offset)
 {
 	uint16_t i;
 
@@ -280,68 +316,69 @@ cperf_set_ops_aead(struct rte_crypto_op **ops,
 		sym_op->m_src = bufs_in[i];
 		sym_op->m_dst = bufs_out[i];
 
-		/* cipher parameters */
-		sym_op->cipher.iv.data = test_vector->iv.data;
-		sym_op->cipher.iv.phys_addr = test_vector->iv.phys_addr;
-		sym_op->cipher.iv.length = test_vector->iv.length;
-
-		sym_op->cipher.data.length = options->test_buffer_size;
-		sym_op->cipher.data.offset =
-				RTE_ALIGN_CEIL(options->auth_aad_sz, 16);
+		/* AEAD parameters */
+		sym_op->aead.data.length = options->test_buffer_size;
+		sym_op->aead.data.offset =
+				RTE_ALIGN_CEIL(options->aead_aad_sz, 16);
 
-		sym_op->auth.aad.data = rte_pktmbuf_mtod(bufs_in[i], uint8_t *);
-		sym_op->auth.aad.phys_addr = rte_pktmbuf_mtophys(bufs_in[i]);
-		sym_op->auth.aad.length = options->auth_aad_sz;
+		sym_op->aead.aad.data = rte_pktmbuf_mtod(bufs_in[i], uint8_t *);
+		sym_op->aead.aad.phys_addr = rte_pktmbuf_mtophys(bufs_in[i]);
 
-		/* authentication parameters */
-		if (options->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) {
-			sym_op->auth.digest.data = test_vector->digest.data;
-			sym_op->auth.digest.phys_addr =
+		if (options->aead_op == RTE_CRYPTO_AEAD_OP_DECRYPT) {
+			sym_op->aead.digest.data = test_vector->digest.data;
+			sym_op->aead.digest.phys_addr =
 					test_vector->digest.phys_addr;
-			sym_op->auth.digest.length = options->auth_digest_sz;
 		} else {
 
-			uint32_t offset = sym_op->cipher.data.length +
-						sym_op->cipher.data.offset;
+			uint32_t offset = sym_op->aead.data.length +
+						sym_op->aead.data.offset;
 			struct rte_mbuf *buf, *tbuf;
 
 			if (options->out_of_place) {
 				buf =  bufs_out[i];
 			} else {
-				buf =  bufs_in[i];
-
-				tbuf = buf;
+				tbuf =  bufs_in[i];
 				while ((tbuf->next != NULL) &&
 						(offset >= tbuf->data_len)) {
 					offset -= tbuf->data_len;
 					tbuf = tbuf->next;
 				}
+				buf = tbuf;
 			}
 
-			sym_op->auth.digest.data = rte_pktmbuf_mtod_offset(buf,
+			sym_op->aead.digest.data = rte_pktmbuf_mtod_offset(buf,
 					uint8_t *, offset);
-			sym_op->auth.digest.phys_addr =
+			sym_op->aead.digest.phys_addr =
 					rte_pktmbuf_mtophys_offset(buf,	offset);
-
-			sym_op->auth.digest.length = options->auth_digest_sz;
 		}
+	}
 
-		sym_op->auth.data.length = options->test_buffer_size;
-		sym_op->auth.data.offset = options->auth_aad_sz;
+	if (options->test == CPERF_TEST_TYPE_VERIFY) {
+		for (i = 0; i < nb_ops; i++) {
+			uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i],
+					uint8_t *, iv_offset);
+
+			memcpy(iv_ptr, test_vector->aead_iv.data,
+					test_vector->aead_iv.length);
+		}
 	}
 
 	return 0;
 }
 
 static struct rte_cryptodev_sym_session *
-cperf_create_session(uint8_t dev_id,
+cperf_create_session(struct rte_mempool *sess_mp,
+	uint8_t dev_id,
 	const struct cperf_options *options,
-	const struct cperf_test_vector *test_vector)
+	const struct cperf_test_vector *test_vector,
+	uint16_t iv_offset)
 {
 	struct rte_crypto_sym_xform cipher_xform;
 	struct rte_crypto_sym_xform auth_xform;
+	struct rte_crypto_sym_xform aead_xform;
 	struct rte_cryptodev_sym_session *sess = NULL;
 
+	sess = rte_cryptodev_sym_session_create(sess_mp);
 	/*
 	 * cipher only
 	 */
@@ -350,6 +387,7 @@ cperf_create_session(uint8_t dev_id,
 		cipher_xform.next = NULL;
 		cipher_xform.cipher.algo = options->cipher_algo;
 		cipher_xform.cipher.op = options->cipher_op;
+		cipher_xform.cipher.iv.offset = iv_offset;
 
 		/* cipher different than null */
 		if (options->cipher_algo != RTE_CRYPTO_CIPHER_NULL) {
@@ -357,12 +395,16 @@ cperf_create_session(uint8_t dev_id,
 					test_vector->cipher_key.data;
 			cipher_xform.cipher.key.length =
 					test_vector->cipher_key.length;
+			cipher_xform.cipher.iv.length =
+					test_vector->cipher_iv.length;
 		} else {
 			cipher_xform.cipher.key.data = NULL;
 			cipher_xform.cipher.key.length = 0;
+			cipher_xform.cipher.iv.length = 0;
 		}
 		/* create crypto session */
-		sess = rte_cryptodev_sym_session_create(dev_id,	&cipher_xform);
+		rte_cryptodev_sym_session_init(dev_id, sess, &cipher_xform,
+				sess_mp);
 	/*
 	 *  auth only
 	 */
@@ -375,27 +417,26 @@ cperf_create_session(uint8_t dev_id,
 		/* auth different than null */
 		if (options->auth_algo != RTE_CRYPTO_AUTH_NULL) {
 			auth_xform.auth.digest_length =
-					options->auth_digest_sz;
-			auth_xform.auth.add_auth_data_length =
-					options->auth_aad_sz;
+					options->digest_sz;
 			auth_xform.auth.key.length =
 					test_vector->auth_key.length;
 			auth_xform.auth.key.data = test_vector->auth_key.data;
+			auth_xform.auth.iv.length =
+					test_vector->auth_iv.length;
 		} else {
 			auth_xform.auth.digest_length = 0;
-			auth_xform.auth.add_auth_data_length = 0;
 			auth_xform.auth.key.length = 0;
 			auth_xform.auth.key.data = NULL;
+			auth_xform.auth.iv.length = 0;
 		}
 		/* create crypto session */
-		sess =  rte_cryptodev_sym_session_create(dev_id, &auth_xform);
+		rte_cryptodev_sym_session_init(dev_id, sess, &auth_xform,
+				sess_mp);
 	/*
 	 * cipher and auth
 	 */
 	} else if (options->op_type == CPERF_CIPHER_THEN_AUTH
-			|| options->op_type == CPERF_AUTH_THEN_CIPHER
-			|| options->op_type == CPERF_AEAD) {
-
+			|| options->op_type == CPERF_AUTH_THEN_CIPHER) {
 		/*
 		 * cipher
 		 */
@@ -403,6 +444,7 @@ cperf_create_session(uint8_t dev_id,
 		cipher_xform.next = NULL;
 		cipher_xform.cipher.algo = options->cipher_algo;
 		cipher_xform.cipher.op = options->cipher_op;
+		cipher_xform.cipher.iv.offset = iv_offset;
 
 		/* cipher different than null */
 		if (options->cipher_algo != RTE_CRYPTO_CIPHER_NULL) {
@@ -410,9 +452,12 @@ cperf_create_session(uint8_t dev_id,
 					test_vector->cipher_key.data;
 			cipher_xform.cipher.key.length =
 					test_vector->cipher_key.length;
+			cipher_xform.cipher.iv.length =
+					test_vector->cipher_iv.length;
 		} else {
 			cipher_xform.cipher.key.data = NULL;
 			cipher_xform.cipher.key.length = 0;
+			cipher_xform.cipher.iv.length = 0;
 		}
 
 		/*
@@ -425,56 +470,53 @@ cperf_create_session(uint8_t dev_id,
 
 		/* auth different than null */
 		if (options->auth_algo != RTE_CRYPTO_AUTH_NULL) {
-			auth_xform.auth.digest_length = options->auth_digest_sz;
-			auth_xform.auth.add_auth_data_length =
-					options->auth_aad_sz;
-			/* auth options for aes gcm */
-			if (options->cipher_algo == RTE_CRYPTO_CIPHER_AES_GCM &&
-				options->auth_algo == RTE_CRYPTO_AUTH_AES_GCM) {
-				auth_xform.auth.key.length = 0;
-				auth_xform.auth.key.data = NULL;
-			} else { /* auth options for others */
-				auth_xform.auth.key.length =
+			auth_xform.auth.digest_length = options->digest_sz;
+			auth_xform.auth.iv.length = test_vector->auth_iv.length;
+			auth_xform.auth.key.length =
 					test_vector->auth_key.length;
-				auth_xform.auth.key.data =
-						test_vector->auth_key.data;
-			}
+			auth_xform.auth.key.data =
+					test_vector->auth_key.data;
 		} else {
 			auth_xform.auth.digest_length = 0;
-			auth_xform.auth.add_auth_data_length = 0;
 			auth_xform.auth.key.length = 0;
 			auth_xform.auth.key.data = NULL;
+			auth_xform.auth.iv.length = 0;
 		}
 
-		/* create crypto session for aes gcm */
-		if (options->cipher_algo == RTE_CRYPTO_CIPHER_AES_GCM) {
-			if (options->cipher_op ==
-					RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
-				cipher_xform.next = &auth_xform;
-				/* create crypto session */
-				sess = rte_cryptodev_sym_session_create(dev_id,
-					&cipher_xform);
-			} else { /* decrypt */
-				auth_xform.next = &cipher_xform;
-				/* create crypto session */
-				sess = rte_cryptodev_sym_session_create(dev_id,
-					&auth_xform);
-			}
-		} else { /* create crypto session for other */
-			/* cipher then auth */
-			if (options->op_type == CPERF_CIPHER_THEN_AUTH) {
-				cipher_xform.next = &auth_xform;
-				/* create crypto session */
-				sess = rte_cryptodev_sym_session_create(dev_id,
-						&cipher_xform);
-			} else { /* auth then cipher */
-				auth_xform.next = &cipher_xform;
-				/* create crypto session */
-				sess = rte_cryptodev_sym_session_create(dev_id,
-						&auth_xform);
-			}
+		/* cipher then auth */
+		if (options->op_type == CPERF_CIPHER_THEN_AUTH) {
+			cipher_xform.next = &auth_xform;
+			/* create crypto session */
+			rte_cryptodev_sym_session_init(dev_id,
+					sess, &cipher_xform, sess_mp);
+		} else { /* auth then cipher */
+			auth_xform.next = &cipher_xform;
+			/* create crypto session */
+			rte_cryptodev_sym_session_init(dev_id,
+					sess, &auth_xform, sess_mp);
 		}
+	} else { /* options->op_type == CPERF_AEAD */
+		aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+		aead_xform.next = NULL;
+		aead_xform.aead.algo = options->aead_algo;
+		aead_xform.aead.op = options->aead_op;
+		aead_xform.aead.iv.offset = iv_offset;
+
+		aead_xform.aead.key.data =
+					test_vector->aead_key.data;
+		aead_xform.aead.key.length =
+					test_vector->aead_key.length;
+		aead_xform.aead.iv.length = test_vector->aead_iv.length;
+
+		aead_xform.aead.digest_length = options->digest_sz;
+		aead_xform.aead.aad_length =
+					options->aead_aad_sz;
+
+		/* Create crypto session */
+		rte_cryptodev_sym_session_init(dev_id,
+					sess, &aead_xform, sess_mp);
 	}
+
 	return sess;
 }
 
@@ -486,14 +528,14 @@ cperf_get_op_functions(const struct cperf_options *options,
 
 	op_fns->sess_create = cperf_create_session;
 
-	if (options->op_type == CPERF_AEAD
-			|| options->op_type == CPERF_AUTH_THEN_CIPHER
+	if (options->op_type == CPERF_AEAD) {
+		op_fns->populate_ops = cperf_set_ops_aead;
+		return 0;
+	}
+
+	if (options->op_type == CPERF_AUTH_THEN_CIPHER
 			|| options->op_type == CPERF_CIPHER_THEN_AUTH) {
-		if (options->cipher_algo == RTE_CRYPTO_CIPHER_AES_GCM &&
-				options->auth_algo == RTE_CRYPTO_AUTH_AES_GCM)
-			op_fns->populate_ops = cperf_set_ops_aead;
-		else
-			op_fns->populate_ops = cperf_set_ops_cipher_auth;
+		op_fns->populate_ops = cperf_set_ops_cipher_auth;
 		return 0;
 	}
 	if (options->op_type == CPERF_AUTH_ONLY) {