summaryrefslogtreecommitdiffstats
path: root/drivers/common
diff options
context:
space:
mode:
authorLuca Boccassi <luca.boccassi@gmail.com>2018-11-01 11:59:50 +0000
committerLuca Boccassi <luca.boccassi@gmail.com>2018-11-01 12:00:19 +0000
commit8d01b9cd70a67cdafd5b965a70420c3bd7fb3f82 (patch)
tree208e3bc33c220854d89d010e3abf720a2e62e546 /drivers/common
parentb63264c8342e6a1b6971c79550d2af2024b6a4de (diff)
New upstream version 18.11-rc1upstream/18.11-rc1
Change-Id: Iaa71986dd6332e878d8f4bf493101b2bbc6313bb Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>
Diffstat (limited to 'drivers/common')
-rw-r--r--drivers/common/Makefile15
-rw-r--r--drivers/common/cpt/Makefile25
-rw-r--r--drivers/common/cpt/cpt_common.h91
-rw-r--r--drivers/common/cpt/cpt_hw_types.h522
-rw-r--r--drivers/common/cpt/cpt_mcode_defines.h386
-rw-r--r--drivers/common/cpt/cpt_pmd_logs.h50
-rw-r--r--drivers/common/cpt/cpt_pmd_ops_helper.c41
-rw-r--r--drivers/common/cpt/cpt_pmd_ops_helper.h34
-rw-r--r--drivers/common/cpt/cpt_request_mgr.h185
-rw-r--r--drivers/common/cpt/cpt_ucode.h3648
-rw-r--r--drivers/common/cpt/meson.build8
-rw-r--r--drivers/common/cpt/rte_common_cpt_version.map6
-rw-r--r--drivers/common/dpaax/Makefile31
-rw-r--r--drivers/common/dpaax/dpaax_iova_table.c465
-rw-r--r--drivers/common/dpaax/dpaax_iova_table.h105
-rw-r--r--drivers/common/dpaax/dpaax_logs.h39
-rw-r--r--drivers/common/dpaax/meson.build12
-rw-r--r--drivers/common/dpaax/rte_common_dpaax_version.map11
-rw-r--r--drivers/common/meson.build2
-rw-r--r--drivers/common/mvep/Makefile38
-rw-r--r--drivers/common/mvep/meson.build19
-rw-r--r--drivers/common/mvep/mvep_common.c45
-rw-r--r--drivers/common/mvep/rte_common_mvep_version.map6
-rw-r--r--drivers/common/mvep/rte_mvep_common.h21
-rw-r--r--drivers/common/qat/qat_common.h3
-rw-r--r--drivers/common/qat/qat_device.c60
-rw-r--r--drivers/common/qat/qat_device.h12
-rw-r--r--drivers/common/qat/qat_qp.c48
-rw-r--r--drivers/common/qat/qat_qp.h1
29 files changed, 5908 insertions, 21 deletions
diff --git a/drivers/common/Makefile b/drivers/common/Makefile
index 0fd22376..87b8a59a 100644
--- a/drivers/common/Makefile
+++ b/drivers/common/Makefile
@@ -4,8 +4,23 @@
include $(RTE_SDK)/mk/rte.vars.mk
+ifeq ($(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO),y)
+DIRS-y += cpt
+endif
+
ifeq ($(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF)$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL),yy)
DIRS-y += octeontx
endif
+MVEP-y := $(CONFIG_RTE_LIBRTE_MVPP2_PMD)
+MVEP-y += $(CONFIG_RTE_LIBRTE_MVNETA_PMD)
+MVEP-y += $(CONFIG_RTE_LIBRTE_PMD_MVSAM_CRYPTO)
+ifneq (,$(findstring y,$(MVEP-y)))
+DIRS-y += mvep
+endif
+
+ifeq ($(CONFIG_RTE_LIBRTE_COMMON_DPAAX),y)
+DIRS-y += dpaax
+endif
+
include $(RTE_SDK)/mk/rte.subdir.mk
diff --git a/drivers/common/cpt/Makefile b/drivers/common/cpt/Makefile
new file mode 100644
index 00000000..2340aa96
--- /dev/null
+++ b/drivers/common/cpt/Makefile
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium, Inc
+#
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_common_cpt.a
+
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -I$(RTE_SDK)/drivers/bus/pci
+EXPORT_MAP := rte_common_cpt_version.map
+
+LIBABIVER := 1
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-y += cpt_pmd_ops_helper.c
+
+LDLIBS += -lrte_eal
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/common/cpt/cpt_common.h b/drivers/common/cpt/cpt_common.h
new file mode 100644
index 00000000..8461cd60
--- /dev/null
+++ b/drivers/common/cpt/cpt_common.h
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_COMMON_H_
+#define _CPT_COMMON_H_
+
+/*
+ * This file defines common macros and structs
+ */
+
+/*
+ * Macros to determine CPT model. Driver makefile will define CPT_MODEL
+ * accordingly
+ */
+#define CRYPTO_OCTEONTX 0x1
+
+#define TIME_IN_RESET_COUNT 5
+
+/* Default command timeout in seconds */
+#define DEFAULT_COMMAND_TIMEOUT 4
+
+#define CPT_COUNT_THOLD 32
+#define CPT_TIMER_THOLD 0x3F
+
+#define AE_TYPE 1
+#define SE_TYPE 2
+
+#ifndef ROUNDUP4
+#define ROUNDUP4(val) (((val) + 3) & 0xfffffffc)
+#endif
+
+#ifndef ROUNDUP8
+#define ROUNDUP8(val) (((val) + 7) & 0xfffffff8)
+#endif
+
+#ifndef ROUNDUP16
+#define ROUNDUP16(val) (((val) + 15) & 0xfffffff0)
+#endif
+
+#ifndef __hot
+#define __hot __attribute__((hot))
+#endif
+
+#define MOD_INC(i, l) ((i) == (l - 1) ? (i) = 0 : (i)++)
+
+struct cptvf_meta_info {
+ void *cptvf_meta_pool;
+ int cptvf_op_mlen;
+ int cptvf_op_sb_mlen;
+};
+
+struct rid {
+ /** Request id of a crypto operation */
+ uintptr_t rid;
+};
+
+/*
+ * Pending queue structure
+ *
+ */
+struct pending_queue {
+ /** Tail of queue to be used for enqueue */
+ uint16_t enq_tail;
+ /** Head of queue to be used for dequeue */
+ uint16_t deq_head;
+ /** Array of pending requests */
+ struct rid *rid_queue;
+ /** Pending requests count */
+ uint64_t pending_count;
+};
+
+struct cpt_request_info {
+ /** Data path fields */
+ uint64_t comp_baddr;
+ volatile uint64_t *completion_addr;
+ volatile uint64_t *alternate_caddr;
+ void *op;
+ struct {
+ uint64_t ei0;
+ uint64_t ei1;
+ uint64_t ei2;
+ uint64_t ei3;
+ } ist;
+
+ /** Control path fields */
+ uint64_t time_out;
+ uint8_t extra_time;
+};
+
+#endif /* _CPT_COMMON_H_ */
diff --git a/drivers/common/cpt/cpt_hw_types.h b/drivers/common/cpt/cpt_hw_types.h
new file mode 100644
index 00000000..cff59c79
--- /dev/null
+++ b/drivers/common/cpt/cpt_hw_types.h
@@ -0,0 +1,522 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_HW_TYPES_H_
+#define _CPT_HW_TYPES_H_
+
+#include <rte_byteorder.h>
+
+/*
+ * This file defines HRM specific structs.
+ *
+ */
+
+#define CPT_VF_INTR_MBOX_MASK (1<<0)
+#define CPT_VF_INTR_DOVF_MASK (1<<1)
+#define CPT_VF_INTR_IRDE_MASK (1<<2)
+#define CPT_VF_INTR_NWRP_MASK (1<<3)
+#define CPT_VF_INTR_SWERR_MASK (1<<4)
+#define CPT_VF_INTR_HWERR_MASK (1<<5)
+#define CPT_VF_INTR_FAULT_MASK (1<<6)
+
+#define CPT_INST_SIZE (64)
+#define CPT_NEXT_CHUNK_PTR_SIZE (8)
+
+/*
+ * CPT_INST_S software command definitions
+ * Words EI (0-3)
+ */
+typedef union {
+ uint64_t u64;
+ struct {
+ uint16_t opcode;
+ uint16_t param1;
+ uint16_t param2;
+ uint16_t dlen;
+ } s;
+} vq_cmd_word0_t;
+
+typedef union {
+ uint64_t u64;
+ struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ uint64_t grp : 3;
+ uint64_t cptr : 61;
+#else
+ uint64_t cptr : 61;
+ uint64_t grp : 3;
+#endif
+ } s;
+} vq_cmd_word3_t;
+
+typedef struct cpt_vq_command {
+ vq_cmd_word0_t cmd;
+ uint64_t dptr;
+ uint64_t rptr;
+ vq_cmd_word3_t cptr;
+} cpt_vq_cmd_t;
+
+/**
+ * Structure cpt_inst_s
+ *
+ * CPT Instruction Structure
+ * This structure specifies the instruction layout.
+ * Instructions are stored in memory as little-endian unless
+ * CPT()_PF_Q()_CTL[INST_BE] is set.
+ */
+typedef union cpt_inst_s {
+ uint64_t u[8];
+ struct cpt_inst_s_8s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_17_63 : 47;
+ /* [ 16: 16] Done interrupt.
+ * 0 = No interrupts related to this instruction.
+ * 1 = When the instruction completes,CPT()_VQ()_DONE[DONE]
+ * will be incremented, and based on the rules described
+ * there an interrupt may occur.
+ */
+ uint64_t doneint : 1;
+ uint64_t reserved_0_15 : 16;
+#else /* Word 0 - Little Endian */
+ uint64_t reserved_0_15 : 16;
+ uint64_t doneint : 1;
+ uint64_t reserved_17_63 : 47;
+#endif /* Word 0 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 1 - Big Endian */
+ /* [127: 64] Result IOVA.
+ * If nonzero, specifies where to write CPT_RES_S.
+ * If zero, no result structure will be written.
+ * Address must be 16-byte aligned.
+ *
+ * Bits <63:49> are ignored by hardware; software should
+ * use a sign-extended bit <48> for forward compatibility.
+ */
+ uint64_t res_addr : 64;
+#else /* Word 1 - Little Endian */
+ uint64_t res_addr : 64;
+#endif /* Word 1 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 2 - Big Endian */
+ uint64_t reserved_172_191 : 20;
+ /* [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to
+ * use when CPT submits work to SSO.
+ * For the SSO to not discard the add-work request, FPA_PF_MAP()
+ * must map [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid.
+ */
+ uint64_t grp : 10;
+ /* [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use
+ * when CPT submits work to SSO.
+ */
+ uint64_t tt : 2;
+ /* [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when
+ * CPT submits work to SSO.
+ */
+ uint64_t tag : 32;
+#else /* Word 2 - Little Endian */
+ uint64_t tag : 32;
+ uint64_t tt : 2;
+ uint64_t grp : 10;
+ uint64_t reserved_172_191 : 20;
+#endif /* Word 2 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 3 - Big Endian */
+ /** [255:192] If [WQ_PTR] is nonzero, it is a pointer to a
+ * work-queue entry that CPT submits work to SSO after all
+ * context, output data, and result write operations are
+ * visible to other CNXXXX units and the cores.
+ * Bits <2:0> must be zero.
+ * Bits <63:49> are ignored by hardware; software should use a
+ * sign-extended bit <48> for forward compatibility.
+ * Internal:Bits <63:49>, <2:0> are ignored by hardware,
+ * treated as always 0x0.
+ **/
+ uint64_t wq_ptr : 64;
+#else /* Word 3 - Little Endian */
+ uint64_t wq_ptr : 64;
+#endif /* Word 3 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 4 - Big Endian */
+ union {
+ /** [319:256] Engine instruction word 0. Passed to the
+ * AE/SE.
+ **/
+ uint64_t ei0 : 64;
+ vq_cmd_word0_t vq_cmd_w0;
+ };
+#else /* Word 4 - Little Endian */
+ union {
+ uint64_t ei0 : 64;
+ vq_cmd_word0_t vq_cmd_w0;
+ };
+#endif /* Word 4 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 5 - Big Endian */
+ union {
+ /** [383:320] Engine instruction word 1. Passed to the
+ * AE/SE.
+ **/
+ uint64_t ei1 : 64;
+ uint64_t dptr;
+ };
+#else /* Word 5 - Little Endian */
+ union {
+ uint64_t ei1 : 64;
+ uint64_t dptr;
+ };
+#endif /* Word 5 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 6 - Big Endian */
+ union {
+ /** [447:384] Engine instruction word 2. Passed to the
+ * AE/SE.
+ **/
+ uint64_t ei2 : 64;
+ uint64_t rptr;
+ };
+#else /* Word 6 - Little Endian */
+ union {
+ uint64_t ei2 : 64;
+ uint64_t rptr;
+ };
+#endif /* Word 6 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 7 - Big Endian */
+ union {
+ /** [511:448] Engine instruction word 3. Passed to the
+ * AE/SE.
+ **/
+ uint64_t ei3 : 64;
+ vq_cmd_word3_t vq_cmd_w3;
+ };
+#else /* Word 7 - Little Endian */
+ union {
+ uint64_t ei3 : 64;
+ vq_cmd_word3_t vq_cmd_w3;
+ };
+#endif /* Word 7 - End */
+ } s8x;
+} cpt_inst_s_t;
+
+/**
+ * Structure cpt_res_s
+ *
+ * CPT Result Structure
+ * The CPT coprocessor writes the result structure after it completes a
+ * CPT_INST_S instruction. The result structure is exactly 16 bytes, and each
+ * instruction completion produces exactly one result structure.
+ *
+ * This structure is stored in memory as little-endian unless
+ * CPT()_PF_Q()_CTL[INST_BE] is set.
+ */
+typedef union cpt_res_s {
+ uint64_t u[2];
+ struct cpt_res_s_8s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_17_63 : 47;
+ /** [ 16: 16] Done interrupt. This bit is copied from the
+ * corresponding instruction's CPT_INST_S[DONEINT].
+ **/
+ uint64_t doneint : 1;
+ uint64_t reserved_8_15 : 8;
+ /** [ 7: 0] Indicates completion/error status of the CPT
+ * coprocessor for the associated instruction, as enumerated by
+ * CPT_COMP_E. Core software may write the memory location
+ * containing [COMPCODE] to 0x0 before ringing the doorbell, and
+ * then poll for completion by checking for a nonzero value.
+ *
+ * Once the core observes a nonzero [COMPCODE] value in this
+ * case, the CPT coprocessor will have also completed L2/DRAM
+ * write operations.
+ **/
+ uint64_t compcode : 8;
+#else /* Word 0 - Little Endian */
+ uint64_t compcode : 8;
+ uint64_t reserved_8_15 : 8;
+ uint64_t doneint : 1;
+ uint64_t reserved_17_63 : 47;
+#endif /* Word 0 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 1 - Big Endian */
+ uint64_t reserved_64_127 : 64;
+#else /* Word 1 - Little Endian */
+ uint64_t reserved_64_127 : 64;
+#endif /* Word 1 - End */
+ } s8x;
+} cpt_res_s_t;
+
+/**
+ * Register (NCB) cpt#_vq#_ctl
+ *
+ * CPT VF Queue Control Registers
+ * This register configures queues. This register should be changed (other than
+ * clearing [ENA]) only when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]).
+ */
+typedef union {
+ uint64_t u;
+ struct cptx_vqx_ctl_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_1_63 : 63;
+ /** [ 0: 0](R/W/H) Enables the logical instruction queue.
+ * See also CPT()_PF_Q()_CTL[CONT_ERR] and
+ * CPT()_VQ()_INPROG[INFLIGHT].
+ * 1 = Queue is enabled.
+ * 0 = Queue is disabled.
+ **/
+ uint64_t ena : 1;
+#else /* Word 0 - Little Endian */
+ uint64_t ena : 1;
+ uint64_t reserved_1_63 : 63;
+#endif /* Word 0 - End */
+ } s;
+} cptx_vqx_ctl_t;
+
+/**
+ * Register (NCB) cpt#_vq#_done
+ *
+ * CPT Queue Done Count Registers
+ * These registers contain the per-queue instruction done count.
+ */
+typedef union {
+ uint64_t u;
+ struct cptx_vqx_done_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_20_63 : 44;
+ /** [ 19: 0](R/W/H) Done count. When CPT_INST_S[DONEINT] set
+ * and that instruction completes,CPT()_VQ()_DONE[DONE] is
+ * incremented when the instruction finishes. Write to this
+ * field are for diagnostic use only; instead software writes
+ * CPT()_VQ()_DONE_ACK with the number of decrements for this
+ * field.
+ *
+ * Interrupts are sent as follows:
+ *
+ * When CPT()_VQ()_DONE[DONE] = 0, then no results are pending,
+ * the interrupt coalescing timer is held to zero, and an
+ * interrupt is not sent.
+ *
+ * When CPT()_VQ()_DONE[DONE] != 0, then the interrupt
+ * coalescing timer counts. If the counter is >= CPT()_VQ()_DONE
+ * _WAIT[TIME_WAIT]*1024, or CPT()_VQ()_DONE[DONE] >= CPT()_VQ()
+ * _DONE_WAIT[NUM_WAIT], i.e. enough time has passed or enough
+ * results have arrived, then the interrupt is sent. Otherwise,
+ * it is not sent due to coalescing.
+ *
+ * When CPT()_VQ()_DONE_ACK is written (or CPT()_VQ()_DONE is
+ * written but this is not typical), the interrupt coalescing
+ * timer restarts. Note after decrementing this interrupt
+ * equation is recomputed, for example if CPT()_VQ()_DONE[DONE]
+ * >= CPT()_VQ()_DONE_WAIT[NUM_WAIT] and because the timer is
+ * zero, the interrupt will be resent immediately. (This covers
+ * the race case between software acknowledging an interrupt and
+ * a result returning.)
+ *
+ * When CPT()_VQ()_DONE_ENA_W1S[DONE] = 0, interrupts are not
+ * sent, but the counting described above still occurs.
+ *
+ * Since CPT instructions complete out-of-order, if software is
+ * using completion interrupts the suggested scheme is to
+ * request a DONEINT on each request, and when an interrupt
+ * arrives perform a "greedy" scan for completions; even if a
+ * later command is acknowledged first this will not result in
+ * missing a completion.
+ *
+ * Software is responsible for making sure [DONE] does not
+ * overflow; for example by insuring there are not more than
+ * 2^20-1 instructions in flight that may request interrupts.
+ **/
+ uint64_t done : 20;
+#else /* Word 0 - Little Endian */
+ uint64_t done : 20;
+ uint64_t reserved_20_63 : 44;
+#endif /* Word 0 - End */
+ } s;
+} cptx_vqx_done_t;
+
+/**
+ * Register (NCB) cpt#_vq#_done_ack
+ *
+ * CPT Queue Done Count Ack Registers
+ * This register is written by software to acknowledge interrupts.
+ */
+typedef union {
+ uint64_t u;
+ struct cptx_vqx_done_ack_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_20_63 : 44;
+ /** [ 19: 0](R/W/H) Number of decrements to CPT()_VQ()_DONE
+ * [DONE]. Reads CPT()_VQ()_DONE[DONE].
+ *
+ * Written by software to acknowledge interrupts. If CPT()_VQ()_
+ * DONE[DONE] is still nonzero the interrupt will be re-sent if
+ * the conditions described in CPT()_VQ()_DONE[DONE] are
+ * satisfied.
+ **/
+ uint64_t done_ack : 20;
+#else /* Word 0 - Little Endian */
+ uint64_t done_ack : 20;
+ uint64_t reserved_20_63 : 44;
+#endif /* Word 0 - End */
+ } s;
+} cptx_vqx_done_ack_t;
+
+/**
+ * Register (NCB) cpt#_vq#_done_wait
+ *
+ * CPT Queue Done Interrupt Coalescing Wait Registers
+ * Specifies the per queue interrupt coalescing settings.
+ */
+typedef union {
+ uint64_t u;
+ struct cptx_vqx_done_wait_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_48_63 : 16;
+ /** [ 47: 32](R/W) Time hold-off. When CPT()_VQ()_DONE[DONE] =
+ * 0, or CPT()_VQ()_DONE_ACK is written a timer is cleared. When
+ * the timer reaches [TIME_WAIT]*1024 then interrupt coalescing
+ * ends; see CPT()_VQ()_DONE[DONE]. If 0x0, time coalescing is
+ * disabled.
+ **/
+ uint64_t time_wait : 16;
+ uint64_t reserved_20_31 : 12;
+ /** [ 19: 0](R/W) Number of messages hold-off. When
+ * CPT()_VQ()_DONE[DONE] >= [NUM_WAIT] then interrupt coalescing
+ * ends; see CPT()_VQ()_DONE[DONE]. If 0x0, same behavior as
+ * 0x1.
+ **/
+ uint64_t num_wait : 20;
+#else /* Word 0 - Little Endian */
+ uint64_t num_wait : 20;
+ uint64_t reserved_20_31 : 12;
+ uint64_t time_wait : 16;
+ uint64_t reserved_48_63 : 16;
+#endif /* Word 0 - End */
+ } s;
+} cptx_vqx_done_wait_t;
+
+/**
+ * Register (NCB) cpt#_vq#_doorbell
+ *
+ * CPT Queue Doorbell Registers
+ * Doorbells for the CPT instruction queues.
+ */
+typedef union {
+ uint64_t u;
+ struct cptx_vqx_doorbell_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_20_63 : 44;
+ uint64_t dbell_cnt : 20;
+ /** [ 19: 0](R/W/H) Number of instruction queue 64-bit words
+ * to add to the CPT instruction doorbell count. Readback value
+ * is the the current number of pending doorbell requests.
+ *
+ * If counter overflows CPT()_VQ()_MISC_INT[DBELL_DOVF] is set.
+ *
+ * To reset the count back to zero, write one to clear
+ * CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF], then write a value
+ * of 2^20 minus the read [DBELL_CNT], then write one to
+ * CPT()_VQ()_MISC_INT_W1C[DBELL_DOVF] and
+ * CPT()_VQ()_MISC_INT_ENA_W1S[DBELL_DOVF].
+ *
+ * Must be a multiple of 8. All CPT instructions are 8 words
+ * and require a doorbell count of multiple of 8.
+ **/
+#else /* Word 0 - Little Endian */
+ uint64_t dbell_cnt : 20;
+ uint64_t reserved_20_63 : 44;
+#endif /* Word 0 - End */
+ } s;
+} cptx_vqx_doorbell_t;
+
+/**
+ * Register (NCB) cpt#_vq#_inprog
+ *
+ * CPT Queue In Progress Count Registers
+ * These registers contain the per-queue instruction in flight registers.
+ */
+typedef union {
+ uint64_t u;
+ struct cptx_vqx_inprog_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_8_63 : 56;
+ /** [ 7: 0](RO/H) Inflight count. Counts the number of
+ * instructions for the VF for which CPT is fetching, executing
+ * or responding to instructions. However this does not include
+ * any interrupts that are awaiting software handling
+ * (CPT()_VQ()_DONE[DONE] != 0x0).
+ *
+ * A queue may not be reconfigured until:
+ * 1. CPT()_VQ()_CTL[ENA] is cleared by software.
+ * 2. [INFLIGHT] is polled until equals to zero.
+ **/
+ uint64_t inflight : 8;
+#else /* Word 0 - Little Endian */
+ uint64_t inflight : 8;
+ uint64_t reserved_8_63 : 56;
+#endif /* Word 0 - End */
+ } s;
+} cptx_vqx_inprog_t;
+
+/**
+ * Register (NCB) cpt#_vq#_misc_int
+ *
+ * CPT Queue Misc Interrupt Register
+ * These registers contain the per-queue miscellaneous interrupts.
+ */
+typedef union {
+ uint64_t u;
+ struct cptx_vqx_misc_int_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_7_63 : 57;
+ /** [ 6: 6](R/W1C/H) Translation fault detected. */
+ uint64_t fault : 1;
+ /** [ 5: 5](R/W1C/H) Hardware error from engines. */
+ uint64_t hwerr : 1;
+ /** [ 4: 4](R/W1C/H) Software error from engines. */
+ uint64_t swerr : 1;
+ /** [ 3: 3](R/W1C/H) NCB result write response error. */
+ uint64_t nwrp : 1;
+ /** [ 2: 2](R/W1C/H) Instruction NCB read response error. */
+ uint64_t irde : 1;
+ /** [ 1: 1](R/W1C/H) Doorbell overflow. */
+ uint64_t dovf : 1;
+ /** [ 0: 0](R/W1C/H) PF to VF mailbox interrupt. Set when
+ * CPT()_VF()_PF_MBOX(0) is written.
+ **/
+ uint64_t mbox : 1;
+#else /* Word 0 - Little Endian */
+ uint64_t mbox : 1;
+ uint64_t dovf : 1;
+ uint64_t irde : 1;
+ uint64_t nwrp : 1;
+ uint64_t swerr : 1;
+ uint64_t hwerr : 1;
+ uint64_t fault : 1;
+ uint64_t reserved_5_63 : 59;
+#endif /* Word 0 - End */
+ } s;
+} cptx_vqx_misc_int_t;
+
+/**
+ * Register (NCB) cpt#_vq#_saddr
+ *
+ * CPT Queue Starting Buffer Address Registers
+ * These registers set the instruction buffer starting address.
+ */
+typedef union {
+ uint64_t u;
+ struct cptx_vqx_saddr_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+ uint64_t reserved_49_63 : 15;
+ /** [ 48: 6](R/W/H) Instruction buffer IOVA <48:6>
+ * (64-byte aligned). When written, it is the initial buffer
+ * starting address; when read, it is the next read pointer to
+ * be requested from L2C. The PTR field is overwritten with the
+ * next pointer each time that the command buffer segment is
+ * exhausted. New commands will then be read from the newly
+ * specified command buffer pointer.
+ **/
+ uint64_t ptr : 43;
+ uint64_t reserved_0_5 : 6;
+#else /* Word 0 - Little Endian */
+ uint64_t reserved_0_5 : 6;
+ uint64_t ptr : 43;
+ uint64_t reserved_49_63 : 15;
+#endif /* Word 0 - End */
+ } s;
+} cptx_vqx_saddr_t;
+
+#endif /*_CPT_HW_TYPES_H_ */
diff --git a/drivers/common/cpt/cpt_mcode_defines.h b/drivers/common/cpt/cpt_mcode_defines.h
new file mode 100644
index 00000000..becc14f4
--- /dev/null
+++ b/drivers/common/cpt/cpt_mcode_defines.h
@@ -0,0 +1,386 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_MCODE_DEFINES_H_
+#define _CPT_MCODE_DEFINES_H_
+
+#include <rte_byteorder.h>
+#include <rte_memory.h>
+
+/*
+ * This file defines macros and structures according to microcode spec
+ *
+ */
+/* SE opcodes */
+#define CPT_MAJOR_OP_FC 0x33
+#define CPT_MAJOR_OP_HASH 0x34
+#define CPT_MAJOR_OP_HMAC 0x35
+#define CPT_MAJOR_OP_ZUC_SNOW3G 0x37
+#define CPT_MAJOR_OP_KASUMI 0x38
+#define CPT_MAJOR_OP_MISC 0x01
+
+#define CPT_BYTE_16 16
+#define CPT_BYTE_24 24
+#define CPT_BYTE_32 32
+#define CPT_MAX_SG_IN_OUT_CNT 32
+#define CPT_MAX_SG_CNT (CPT_MAX_SG_IN_OUT_CNT/2)
+
+#define COMPLETION_CODE_SIZE 8
+#define COMPLETION_CODE_INIT 0
+
+#define SG_LIST_HDR_SIZE (8u)
+#define SG_ENTRY_SIZE sizeof(sg_comp_t)
+
+#define CPT_DMA_MODE (1 << 7)
+
+#define CPT_FROM_CTX 0
+#define CPT_FROM_DPTR 1
+
+#define FC_GEN 0x1
+#define ZUC_SNOW3G 0x2
+#define KASUMI 0x3
+#define HASH_HMAC 0x4
+
+#define ZS_EA 0x1
+#define ZS_IA 0x2
+#define K_F8 0x4
+#define K_F9 0x8
+
+#define CPT_OP_CIPHER_ENCRYPT 0x1
+#define CPT_OP_CIPHER_DECRYPT 0x2
+#define CPT_OP_CIPHER_MASK 0x3
+
+#define CPT_OP_AUTH_VERIFY 0x4
+#define CPT_OP_AUTH_GENERATE 0x8
+#define CPT_OP_AUTH_MASK 0xC
+
+#define CPT_OP_ENCODE (CPT_OP_CIPHER_ENCRYPT | CPT_OP_AUTH_GENERATE)
+#define CPT_OP_DECODE (CPT_OP_CIPHER_DECRYPT | CPT_OP_AUTH_VERIFY)
+
+/* #define CPT_ALWAYS_USE_SG_MODE */
+#define CPT_ALWAYS_USE_SEPARATE_BUF
+
+/*
+ * Parameters for Flexi Crypto
+ * requests
+ */
+#define VALID_AAD_BUF 0x01
+#define VALID_MAC_BUF 0x02
+#define VALID_IV_BUF 0x04
+#define SINGLE_BUF_INPLACE 0x08
+#define SINGLE_BUF_HEADTAILROOM 0x10
+
+#define ENCR_IV_OFFSET(__d_offs) ((__d_offs >> 32) & 0xffff)
+#define ENCR_OFFSET(__d_offs) ((__d_offs >> 16) & 0xffff)
+#define AUTH_OFFSET(__d_offs) (__d_offs & 0xffff)
+#define ENCR_DLEN(__d_lens) (__d_lens >> 32)
+#define AUTH_DLEN(__d_lens) (__d_lens & 0xffffffff)
+
+/* FC offset_control at start of DPTR in bytes */
+#define OFF_CTRL_LEN 8 /**< bytes */
+
+typedef enum {
+ MD5_TYPE = 1,
+ SHA1_TYPE = 2,
+ SHA2_SHA224 = 3,
+ SHA2_SHA256 = 4,
+ SHA2_SHA384 = 5,
+ SHA2_SHA512 = 6,
+ GMAC_TYPE = 7,
+ XCBC_TYPE = 8,
+ SHA3_SHA224 = 10,
+ SHA3_SHA256 = 11,
+ SHA3_SHA384 = 12,
+ SHA3_SHA512 = 13,
+ SHA3_SHAKE256 = 14,
+ SHA3_SHAKE512 = 15,
+
+ /* These are only for software use */
+ ZUC_EIA3 = 0x90,
+ SNOW3G_UIA2 = 0x91,
+ KASUMI_F9_CBC = 0x92,
+ KASUMI_F9_ECB = 0x93,
+} mc_hash_type_t;
+
+typedef enum {
+ /* To support passthrough */
+ PASSTHROUGH = 0x0,
+ /*
+ * These are defined by MC for Flexi crypto
+ * for field of 4 bits
+ */
+ DES3_CBC = 0x1,
+ DES3_ECB = 0x2,
+ AES_CBC = 0x3,
+ AES_ECB = 0x4,
+ AES_CFB = 0x5,
+ AES_CTR = 0x6,
+ AES_GCM = 0x7,
+ AES_XTS = 0x8,
+
+ /* These are only for software use */
+ ZUC_EEA3 = 0x90,
+ SNOW3G_UEA2 = 0x91,
+ KASUMI_F8_CBC = 0x92,
+ KASUMI_F8_ECB = 0x93,
+} mc_cipher_type_t;
+
+typedef enum {
+ AES_128_BIT = 0x1,
+ AES_192_BIT = 0x2,
+ AES_256_BIT = 0x3
+} mc_aes_type_t;
+
+typedef enum {
+ /* Microcode errors */
+ NO_ERR = 0x00,
+ ERR_OPCODE_UNSUPPORTED = 0x01,
+
+ /* SCATTER GATHER */
+ ERR_SCATTER_GATHER_WRITE_LENGTH = 0x02,
+ ERR_SCATTER_GATHER_LIST = 0x03,
+ ERR_SCATTER_GATHER_NOT_SUPPORTED = 0x04,
+
+ /* SE GC */
+ ERR_GC_LENGTH_INVALID = 0x41,
+ ERR_GC_RANDOM_LEN_INVALID = 0x42,
+ ERR_GC_DATA_LEN_INVALID = 0x43,
+ ERR_GC_DRBG_TYPE_INVALID = 0x44,
+ ERR_GC_CTX_LEN_INVALID = 0x45,
+ ERR_GC_CIPHER_UNSUPPORTED = 0x46,
+ ERR_GC_AUTH_UNSUPPORTED = 0x47,
+ ERR_GC_OFFSET_INVALID = 0x48,
+ ERR_GC_HASH_MODE_UNSUPPORTED = 0x49,
+ ERR_GC_DRBG_ENTROPY_LEN_INVALID = 0x4a,
+ ERR_GC_DRBG_ADDNL_LEN_INVALID = 0x4b,
+ ERR_GC_ICV_MISCOMPARE = 0x4c,
+ ERR_GC_DATA_UNALIGNED = 0x4d,
+
+ /* API Layer */
+ ERR_BAD_ALT_CCODE = 0xfd,
+ ERR_REQ_PENDING = 0xfe,
+ ERR_REQ_TIMEOUT = 0xff,
+
+ ERR_BAD_INPUT_LENGTH = (0x40000000 | 384), /* 0x40000180 */
+ ERR_BAD_KEY_LENGTH,
+ ERR_BAD_KEY_HANDLE,
+ ERR_BAD_CONTEXT_HANDLE,
+ ERR_BAD_SCALAR_LENGTH,
+ ERR_BAD_DIGEST_LENGTH,
+ ERR_BAD_INPUT_ARG,
+ ERR_BAD_RECORD_PADDING,
+ ERR_NB_REQUEST_PENDING,
+ ERR_EIO,
+ ERR_ENODEV,
+} mc_error_code_t;
+
+/**
+ * Enumeration cpt_comp_e
+ *
+ * CPT Completion Enumeration
+ * Enumerates the values of CPT_RES_S[COMPCODE].
+ */
+typedef enum {
+ CPT_8X_COMP_E_NOTDONE = (0x00),
+ CPT_8X_COMP_E_GOOD = (0x01),
+ CPT_8X_COMP_E_FAULT = (0x02),
+ CPT_8X_COMP_E_SWERR = (0x03),
+ CPT_8X_COMP_E_HWERR = (0x04),
+ CPT_8X_COMP_E_LAST_ENTRY = (0xFF)
+} cpt_comp_e_t;
+
+typedef struct sglist_comp {
+ union {
+ uint64_t len;
+ struct {
+ uint16_t len[4];
+ } s;
+ } u;
+ uint64_t ptr[4];
+} sg_comp_t;
+
+struct cpt_sess_misc {
+ /** CPT opcode */
+ uint16_t cpt_op:4;
+ /** ZUC, SNOW3G & KASUMI flags */
+ uint16_t zsk_flag:4;
+ /** Flag for AES GCM */
+ uint16_t aes_gcm:1;
+ /** Flag for AES CTR */
+ uint16_t aes_ctr:1;
+ /** Flag for NULL cipher/auth */
+ uint16_t is_null:1;
+ /** Flag for GMAC */
+ uint16_t is_gmac:1;
+ /** AAD length */
+ uint16_t aad_length;
+ /** MAC len in bytes */
+ uint8_t mac_len;
+ /** IV length in bytes */
+ uint8_t iv_length;
+ /** Auth IV length in bytes */
+ uint8_t auth_iv_length;
+ /** Reserved field */
+ uint8_t rsvd1;
+ /** IV offset in bytes */
+ uint16_t iv_offset;
+ /** Auth IV offset in bytes */
+ uint16_t auth_iv_offset;
+ /** Salt */
+ uint32_t salt;
+ /** Context DMA address */
+ phys_addr_t ctx_dma_addr;
+};
+
+typedef union {
+ uint64_t flags;
+ struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ uint64_t enc_cipher : 4;
+ uint64_t reserved1 : 1;
+ uint64_t aes_key : 2;
+ uint64_t iv_source : 1;
+ uint64_t hash_type : 4;
+ uint64_t reserved2 : 3;
+ uint64_t auth_input_type : 1;
+ uint64_t mac_len : 8;
+ uint64_t reserved3 : 8;
+ uint64_t encr_offset : 16;
+ uint64_t iv_offset : 8;
+ uint64_t auth_offset : 8;
+#else
+ uint64_t auth_offset : 8;
+ uint64_t iv_offset : 8;
+ uint64_t encr_offset : 16;
+ uint64_t reserved3 : 8;
+ uint64_t mac_len : 8;
+ uint64_t auth_input_type : 1;
+ uint64_t reserved2 : 3;
+ uint64_t hash_type : 4;
+ uint64_t iv_source : 1;
+ uint64_t aes_key : 2;
+ uint64_t reserved1 : 1;
+ uint64_t enc_cipher : 4;
+#endif
+ } e;
+} encr_ctrl_t;
+
+typedef struct {
+ encr_ctrl_t enc_ctrl;
+ uint8_t encr_key[32];
+ uint8_t encr_iv[16];
+} mc_enc_context_t;
+
+typedef struct {
+ uint8_t ipad[64];
+ uint8_t opad[64];
+} mc_fc_hmac_context_t;
+
+typedef struct {
+ mc_enc_context_t enc;
+ mc_fc_hmac_context_t hmac;
+} mc_fc_context_t;
+
+typedef struct {
+ uint8_t encr_auth_iv[16];
+ uint8_t ci_key[16];
+ uint8_t zuc_const[32];
+} mc_zuc_snow3g_ctx_t;
+
+typedef struct {
+ uint8_t reg_A[8];
+ uint8_t ci_key[16];
+} mc_kasumi_ctx_t;
+
+struct cpt_ctx {
+ /* Below fields are accessed by sw */
+ uint64_t enc_cipher :8;
+ uint64_t hash_type :8;
+ uint64_t mac_len :8;
+ uint64_t auth_key_len :8;
+ uint64_t fc_type :4;
+ uint64_t hmac :1;
+ uint64_t zsk_flags :3;
+ uint64_t k_ecb :1;
+ uint64_t snow3g :1;
+ uint64_t rsvd :22;
+ /* Below fields are accessed by hardware */
+ union {
+ mc_fc_context_t fctx;
+ mc_zuc_snow3g_ctx_t zs_ctx;
+ mc_kasumi_ctx_t k_ctx;
+ };
+ uint8_t auth_key[64];
+};
+
+/* Buffer pointer */
+typedef struct buf_ptr {
+ void *vaddr;
+ phys_addr_t dma_addr;
+ uint32_t size;
+ uint32_t resv;
+} buf_ptr_t;
+
+/* IOV Pointer */
+typedef struct{
+ int buf_cnt;
+ buf_ptr_t bufs[0];
+} iov_ptr_t;
+
+typedef union opcode_info {
+ uint16_t flags;
+ struct {
+ uint8_t major;
+ uint8_t minor;
+ } s;
+} opcode_info_t;
+
+typedef struct fc_params {
+ /* 0th cache line */
+ union {
+ buf_ptr_t bufs[1];
+ struct {
+ iov_ptr_t *src_iov;
+ iov_ptr_t *dst_iov;
+ };
+ };
+ void *iv_buf;
+ void *auth_iv_buf;
+ buf_ptr_t meta_buf;
+ buf_ptr_t ctx_buf;
+ uint64_t rsvd2;
+
+ /* 1st cache line */
+ buf_ptr_t aad_buf;
+ buf_ptr_t mac_buf;
+
+} fc_params_t;
+
+/*
+ * Parameters for digest
+ * generate requests
+ * Only src_iov, op, ctx_buf, mac_buf, prep_req
+ * meta_buf, auth_data_len are used for digest gen.
+ */
+typedef struct fc_params digest_params_t;
+
+/* Cipher Algorithms */
+typedef mc_cipher_type_t cipher_type_t;
+
+/* Auth Algorithms */
+typedef mc_hash_type_t auth_type_t;
+
+/* Helper macros */
+
+#define CPT_P_ENC_CTRL(fctx) fctx->enc.enc_ctrl.e
+
+#define SRC_IOV_SIZE \
+ (sizeof(iov_ptr_t) + (sizeof(buf_ptr_t) * CPT_MAX_SG_CNT))
+#define DST_IOV_SIZE \
+ (sizeof(iov_ptr_t) + (sizeof(buf_ptr_t) * CPT_MAX_SG_CNT))
+
+#define SESS_PRIV(__sess) \
+ (void *)((uint8_t *)__sess + sizeof(struct cpt_sess_misc))
+
+#endif /* _CPT_MCODE_DEFINES_H_ */
diff --git a/drivers/common/cpt/cpt_pmd_logs.h b/drivers/common/cpt/cpt_pmd_logs.h
new file mode 100644
index 00000000..4cbec4e3
--- /dev/null
+++ b/drivers/common/cpt/cpt_pmd_logs.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_PMD_LOGS_H_
+#define _CPT_PMD_LOGS_H_
+
+#include <rte_log.h>
+
+/*
+ * This file defines log macros
+ */
+
+#define CPT_PMD_DRV_LOG_RAW(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, cpt_logtype, \
+ "cpt: %s(): " fmt "\n", __func__, ##args)
+
+#define CPT_PMD_INIT_FUNC_TRACE() CPT_PMD_DRV_LOG_RAW(DEBUG, " >>")
+
+#define CPT_LOG_INFO(fmt, args...) \
+ CPT_PMD_DRV_LOG_RAW(INFO, fmt, ## args)
+#define CPT_LOG_WARN(fmt, args...) \
+ CPT_PMD_DRV_LOG_RAW(WARNING, fmt, ## args)
+#define CPT_LOG_ERR(fmt, args...) \
+ CPT_PMD_DRV_LOG_RAW(ERR, fmt, ## args)
+
+/*
+ * DP logs, toggled out at compile time if level lower than current level.
+ * DP logs would be logged under 'PMD' type. So for dynamic logging, the
+ * level of 'pmd' has to be used.
+ */
+#define CPT_LOG_DP(level, fmt, args...) \
+ RTE_LOG_DP(level, PMD, fmt "\n", ## args)
+
+#define CPT_LOG_DP_DEBUG(fmt, args...) \
+ CPT_LOG_DP(DEBUG, fmt, ## args)
+#define CPT_LOG_DP_INFO(fmt, args...) \
+ CPT_LOG_DP(INFO, fmt, ## args)
+#define CPT_LOG_DP_WARN(fmt, args...) \
+ CPT_LOG_DP(WARNING, fmt, ## args)
+#define CPT_LOG_DP_ERR(fmt, args...) \
+ CPT_LOG_DP(ERR, fmt, ## args)
+
+/*
+ * cpt_logtype will be used for common logging. This field would be initialized
+ * by otx_* driver routines during PCI probe.
+ */
+int cpt_logtype;
+
+#endif /* _CPT_PMD_LOGS_H_ */
diff --git a/drivers/common/cpt/cpt_pmd_ops_helper.c b/drivers/common/cpt/cpt_pmd_ops_helper.c
new file mode 100644
index 00000000..1c18180f
--- /dev/null
+++ b/drivers/common/cpt/cpt_pmd_ops_helper.c
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#include <rte_common.h>
+
+#include "cpt_common.h"
+#include "cpt_hw_types.h"
+#include "cpt_mcode_defines.h"
+#include "cpt_pmd_ops_helper.h"
+
+#define CPT_MAX_IV_LEN 16
+#define CPT_OFFSET_CONTROL_BYTES 8
+
+int32_t
+cpt_pmd_ops_helper_get_mlen_direct_mode(void)
+{
+ uint32_t len = 0;
+
+ /* Request structure */
+ len = sizeof(struct cpt_request_info);
+
+ /* CPT HW result structure plus extra as it is aligned */
+ len += 2*sizeof(cpt_res_s_t);
+
+ return len;
+}
+
+int
+cpt_pmd_ops_helper_get_mlen_sg_mode(void)
+{
+ uint32_t len = 0;
+
+ len += sizeof(struct cpt_request_info);
+ len += CPT_OFFSET_CONTROL_BYTES + CPT_MAX_IV_LEN;
+ len += ROUNDUP8(SG_LIST_HDR_SIZE +
+ (ROUNDUP4(CPT_MAX_SG_IN_OUT_CNT) >> 2) * SG_ENTRY_SIZE);
+ len += 2 * COMPLETION_CODE_SIZE;
+ len += 2 * sizeof(cpt_res_s_t);
+ return len;
+}
diff --git a/drivers/common/cpt/cpt_pmd_ops_helper.h b/drivers/common/cpt/cpt_pmd_ops_helper.h
new file mode 100644
index 00000000..dd32f9a4
--- /dev/null
+++ b/drivers/common/cpt/cpt_pmd_ops_helper.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_PMD_OPS_HELPER_H_
+#define _CPT_PMD_OPS_HELPER_H_
+
+/*
+ * This file defines the agreement between the common layer and the individual
+ * crypto drivers for OCTEON TX series. Control path in otx* directory can
+ * directly call functions declared here.
+ */
+
+/*
+ * Get meta length required when operating in direct mode (single buffer
+ * in-place)
+ *
+ * @return
+ * - length
+ */
+
+int32_t
+cpt_pmd_ops_helper_get_mlen_direct_mode(void);
+
+/*
+ * Get size of contiguous meta buffer to be allocated when working in scatter
+ * gather mode.
+ *
+ * @return
+ * - length
+ */
+int
+cpt_pmd_ops_helper_get_mlen_sg_mode(void);
+#endif /* _CPT_PMD_OPS_HELPER_H_ */
diff --git a/drivers/common/cpt/cpt_request_mgr.h b/drivers/common/cpt/cpt_request_mgr.h
new file mode 100644
index 00000000..4463cfbe
--- /dev/null
+++ b/drivers/common/cpt/cpt_request_mgr.h
@@ -0,0 +1,185 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_REQUEST_MGR_H_
+#define _CPT_REQUEST_MGR_H_
+
+#include <rte_branch_prediction.h>
+#include <rte_cycles.h>
+
+#include "cpt_common.h"
+#include "cpt_mcode_defines.h"
+
+#if CPT_MODEL == CRYPTO_OCTEONTX
+#include "../../crypto/octeontx/otx_cryptodev_hw_access.h"
+#endif
+
+/*
+ * This file defines the agreement between the common layer and the individual
+ * crypto drivers for OCTEON TX series. Datapath in otx* directory include this
+ * file and all these functions are static inlined for better performance.
+ *
+ */
+
+/*
+ * Get the session size
+ *
+ * This function is used in the data path.
+ *
+ * @return
+ * - session size
+ */
+static __rte_always_inline unsigned int
+cpt_get_session_size(void)
+{
+ unsigned int ctx_len = sizeof(struct cpt_ctx);
+ return (sizeof(struct cpt_sess_misc) + RTE_ALIGN_CEIL(ctx_len, 8));
+}
+
+static __rte_always_inline int32_t __hot
+cpt_enqueue_req(struct cpt_instance *instance, struct pending_queue *pqueue,
+ void *req)
+{
+ struct cpt_request_info *user_req = (struct cpt_request_info *)req;
+ int32_t ret = 0;
+
+ if (unlikely(!req))
+ return 0;
+
+ if (unlikely(pqueue->pending_count >= DEFAULT_CMD_QLEN))
+ return -EAGAIN;
+
+ fill_cpt_inst(instance, req);
+
+ CPT_LOG_DP_DEBUG("req: %p op: %p ", req, user_req->op);
+
+ /* Fill time_out cycles */
+ user_req->time_out = rte_get_timer_cycles() +
+ DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
+ user_req->extra_time = 0;
+
+ /* Default mode of software queue */
+ mark_cpt_inst(instance);
+
+ pqueue->rid_queue[pqueue->enq_tail].rid =
+ (uintptr_t)user_req;
+ /* We will use soft queue length here to limit
+ * requests
+ */
+ MOD_INC(pqueue->enq_tail, DEFAULT_CMD_QLEN);
+ pqueue->pending_count += 1;
+
+ CPT_LOG_DP_DEBUG("Submitted NB cmd with request: %p "
+ "op: %p", user_req, user_req->op);
+
+ return ret;
+}
+
+static __rte_always_inline int __hot
+cpt_pmd_crypto_operation(struct cpt_instance *instance,
+ struct rte_crypto_op *op, struct pending_queue *pqueue,
+ uint8_t cpt_driver_id)
+{
+ struct cpt_sess_misc *sess = NULL;
+ struct rte_crypto_sym_op *sym_op = op->sym;
+ void *prep_req = NULL, *mdata = NULL;
+ int ret = 0;
+ uint64_t cpt_op;
+ struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+
+ if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+ int sess_len;
+
+ sess_len = cpt_get_session_size();
+
+ sess = rte_calloc(__func__, 1, sess_len, 8);
+ if (!sess)
+ return -ENOMEM;
+
+ sess->ctx_dma_addr = rte_malloc_virt2iova(sess) +
+ sizeof(struct cpt_sess_misc);
+
+ ret = instance_session_cfg(sym_op->xform, (void *)sess);
+ if (unlikely(ret))
+ return -EINVAL;
+ } else {
+ sess = (struct cpt_sess_misc *)
+ get_sym_session_private_data(sym_op->session,
+ cpt_driver_id);
+ }
+
+ cpt_op = sess->cpt_op;
+
+ mdata = &(cptvf->meta_info);
+
+ if (likely(cpt_op & CPT_OP_CIPHER_MASK))
+ prep_req = fill_fc_params(op, sess, &mdata, &ret);
+ else
+ prep_req = fill_digest_params(op, sess, &mdata, &ret);
+
+ if (unlikely(!prep_req)) {
+ CPT_LOG_DP_ERR("prep cryto req : op %p, cpt_op 0x%x "
+ "ret 0x%x", op, (unsigned int)cpt_op, ret);
+ goto req_fail;
+ }
+
+ /* Enqueue prepared instruction to HW */
+ ret = cpt_enqueue_req(instance, pqueue, prep_req);
+
+ if (unlikely(ret)) {
+ if (unlikely(ret == -EAGAIN))
+ goto req_fail;
+ CPT_LOG_DP_ERR("Error enqueing crypto request : error "
+ "code %d", ret);
+ goto req_fail;
+ }
+
+ return 0;
+
+req_fail:
+ if (mdata)
+ free_op_meta(mdata, cptvf->meta_info.cptvf_meta_pool);
+ return ret;
+}
+
+static __rte_always_inline int32_t __hot
+cpt_dequeue_burst(struct cpt_instance *instance, uint16_t cnt,
+ void *resp[], uint8_t cc[], struct pending_queue *pqueue)
+{
+ struct cpt_request_info *user_req;
+ struct rid *rid_e;
+ int i, count, pcount;
+ uint8_t ret;
+
+ pcount = pqueue->pending_count;
+ count = (cnt > pcount) ? pcount : cnt;
+
+ for (i = 0; i < count; i++) {
+ rid_e = &pqueue->rid_queue[pqueue->deq_head];
+ user_req = (struct cpt_request_info *)(rid_e->rid);
+
+ if (likely((i+1) < count))
+ rte_prefetch_non_temporal((void *)rid_e[1].rid);
+
+ ret = check_nb_command_id(user_req, instance);
+
+ if (unlikely(ret == ERR_REQ_PENDING)) {
+ /* Stop checking for completions */
+ break;
+ }
+
+ /* Return completion code and op handle */
+ cc[i] = (uint8_t)ret;
+ resp[i] = user_req->op;
+ CPT_LOG_DP_DEBUG("Request %p Op %p completed with code %d",
+ user_req, user_req->op, ret);
+
+ MOD_INC(pqueue->deq_head, DEFAULT_CMD_QLEN);
+ pqueue->pending_count -= 1;
+ }
+
+ return i;
+}
+
+#endif /* _CPT_REQUEST_MGR_H_ */
diff --git a/drivers/common/cpt/cpt_ucode.h b/drivers/common/cpt/cpt_ucode.h
new file mode 100644
index 00000000..c5a9f34b
--- /dev/null
+++ b/drivers/common/cpt/cpt_ucode.h
@@ -0,0 +1,3648 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_UCODE_H_
+#define _CPT_UCODE_H_
+#include <stdbool.h>
+
+#include "cpt_common.h"
+#include "cpt_hw_types.h"
+#include "cpt_mcode_defines.h"
+
+/*
+ * This file defines functions that are interfaces to microcode spec.
+ *
+ */
+
+static uint8_t zuc_d[32] = {
+ 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E,
+ 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF,
+ 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1,
+ 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC
+};
+
+static __rte_always_inline int
+cpt_is_algo_supported(struct rte_crypto_sym_xform *xform)
+{
+ /*
+ * Microcode only supports the following combination.
+ * Encryption followed by authentication
+ * Authentication followed by decryption
+ */
+ if (xform->next) {
+ if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) &&
+ (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
+ (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) {
+ /* Unsupported as of now by microcode */
+ CPT_LOG_DP_ERR("Unsupported combination");
+ return -1;
+ }
+ if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
+ (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) &&
+ (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) {
+ /* For GMAC auth there is no cipher operation */
+ if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM ||
+ xform->next->auth.algo !=
+ RTE_CRYPTO_AUTH_AES_GMAC) {
+ /* Unsupported as of now by microcode */
+ CPT_LOG_DP_ERR("Unsupported combination");
+ return -1;
+ }
+ }
+ }
+ return 0;
+}
+
+static __rte_always_inline void
+gen_key_snow3g(uint8_t *ck, uint32_t *keyx)
+{
+ int i, base;
+
+ for (i = 0; i < 4; i++) {
+ base = 4 * i;
+ keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) |
+ (ck[base + 2] << 8) | (ck[base + 3]);
+ keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]);
+ }
+}
+
+static __rte_always_inline void
+cpt_fc_salt_update(void *ctx,
+ uint8_t *salt)
+{
+ struct cpt_ctx *cpt_ctx = ctx;
+ memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4);
+}
+
+static __rte_always_inline int
+cpt_fc_ciph_validate_key_aes(uint16_t key_len)
+{
+ switch (key_len) {
+ case CPT_BYTE_16:
+ case CPT_BYTE_24:
+ case CPT_BYTE_32:
+ return 0;
+ default:
+ return -1;
+ }
+}
+
+static __rte_always_inline int
+cpt_fc_ciph_validate_key(cipher_type_t type, struct cpt_ctx *cpt_ctx,
+ uint16_t key_len)
+{
+ int fc_type = 0;
+ switch (type) {
+ case PASSTHROUGH:
+ fc_type = FC_GEN;
+ break;
+ case DES3_CBC:
+ case DES3_ECB:
+ fc_type = FC_GEN;
+ break;
+ case AES_CBC:
+ case AES_ECB:
+ case AES_CFB:
+ case AES_CTR:
+ case AES_GCM:
+ if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
+ return -1;
+ fc_type = FC_GEN;
+ break;
+ case AES_XTS:
+ key_len = key_len / 2;
+ if (unlikely(key_len == CPT_BYTE_24)) {
+ CPT_LOG_DP_ERR("Invalid AES key len for XTS");
+ return -1;
+ }
+ if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
+ return -1;
+ fc_type = FC_GEN;
+ break;
+ case ZUC_EEA3:
+ case SNOW3G_UEA2:
+ if (unlikely(key_len != 16))
+ return -1;
+ /* No support for AEAD yet */
+ if (unlikely(cpt_ctx->hash_type))
+ return -1;
+ fc_type = ZUC_SNOW3G;
+ break;
+ case KASUMI_F8_CBC:
+ case KASUMI_F8_ECB:
+ if (unlikely(key_len != 16))
+ return -1;
+ /* No support for AEAD yet */
+ if (unlikely(cpt_ctx->hash_type))
+ return -1;
+ fc_type = KASUMI;
+ break;
+ default:
+ return -1;
+ }
+ return fc_type;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx)
+{
+ cpt_ctx->enc_cipher = 0;
+ CPT_P_ENC_CTRL(fctx).enc_cipher = 0;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len)
+{
+ mc_aes_type_t aes_key_type = 0;
+ switch (key_len) {
+ case CPT_BYTE_16:
+ aes_key_type = AES_128_BIT;
+ break;
+ case CPT_BYTE_24:
+ aes_key_type = AES_192_BIT;
+ break;
+ case CPT_BYTE_32:
+ aes_key_type = AES_256_BIT;
+ break;
+ default:
+ /* This should not happen */
+ CPT_LOG_DP_ERR("Invalid AES key len");
+ return;
+ }
+ CPT_P_ENC_CTRL(fctx).aes_key = aes_key_type;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, uint8_t *key,
+ uint16_t key_len)
+{
+ uint32_t keyx[4];
+ cpt_ctx->snow3g = 1;
+ gen_key_snow3g(key, keyx);
+ memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len);
+ cpt_ctx->fc_type = ZUC_SNOW3G;
+ cpt_ctx->zsk_flags = 0;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, uint8_t *key,
+ uint16_t key_len)
+{
+ cpt_ctx->snow3g = 0;
+ memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len);
+ memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32);
+ cpt_ctx->fc_type = ZUC_SNOW3G;
+ cpt_ctx->zsk_flags = 0;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, uint8_t *key,
+ uint16_t key_len)
+{
+ cpt_ctx->k_ecb = 1;
+ memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
+ cpt_ctx->zsk_flags = 0;
+ cpt_ctx->fc_type = KASUMI;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, uint8_t *key,
+ uint16_t key_len)
+{
+ memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
+ cpt_ctx->zsk_flags = 0;
+ cpt_ctx->fc_type = KASUMI;
+}
+
+static __rte_always_inline int
+cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, uint8_t *key,
+ uint16_t key_len, uint8_t *salt)
+{
+ struct cpt_ctx *cpt_ctx = ctx;
+ mc_fc_context_t *fctx = &cpt_ctx->fctx;
+ uint64_t *ctrl_flags = NULL;
+ int fc_type;
+
+ /* Validate key before proceeding */
+ fc_type = cpt_fc_ciph_validate_key(type, cpt_ctx, key_len);
+ if (unlikely(fc_type == -1))
+ return -1;
+
+ if (fc_type == FC_GEN) {
+ cpt_ctx->fc_type = FC_GEN;
+ ctrl_flags = (uint64_t *)&(fctx->enc.enc_ctrl.flags);
+ *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags);
+ /*
+ * We need to always say IV is from DPTR as user can
+ * sometimes iverride IV per operation.
+ */
+ CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_DPTR;
+ }
+
+ switch (type) {
+ case PASSTHROUGH:
+ cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx);
+ goto fc_success;
+ case DES3_CBC:
+ /* CPT performs DES using 3DES with the 8B DES-key
+ * replicated 2 more times to match the 24B 3DES-key.
+ * Eg. If org. key is "0x0a 0x0b", then new key is
+ * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b"
+ */
+ if (key_len == 8) {
+ /* Skipping the first 8B as it will be copied
+ * in the regular code flow
+ */
+ memcpy(fctx->enc.encr_key+key_len, key, key_len);
+ memcpy(fctx->enc.encr_key+2*key_len, key, key_len);
+ }
+ break;
+ case DES3_ECB:
+ /* For DES3_ECB IV need to be from CTX. */
+ CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_CTX;
+ break;
+ case AES_CBC:
+ case AES_ECB:
+ case AES_CFB:
+ case AES_CTR:
+ cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
+ break;
+ case AES_GCM:
+ /* Even though iv source is from dptr,
+ * aes_gcm salt is taken from ctx
+ */
+ if (salt) {
+ memcpy(fctx->enc.encr_iv, salt, 4);
+ /* Assuming it was just salt update
+ * and nothing else
+ */
+ if (!key)
+ goto fc_success;
+ }
+ cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
+ break;
+ case AES_XTS:
+ key_len = key_len / 2;
+ cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
+
+ /* Copy key2 for XTS into ipad */
+ memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
+ memcpy(fctx->hmac.ipad, &key[key_len], key_len);
+ break;
+ case SNOW3G_UEA2:
+ cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len);
+ goto success;
+ case ZUC_EEA3:
+ cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len);
+ goto success;
+ case KASUMI_F8_ECB:
+ cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len);
+ goto success;
+ case KASUMI_F8_CBC:
+ cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len);
+ goto success;
+ default:
+ break;
+ }
+
+ /* Only for FC_GEN case */
+
+ /* For GMAC auth, cipher must be NULL */
+ if (cpt_ctx->hash_type != GMAC_TYPE)
+ CPT_P_ENC_CTRL(fctx).enc_cipher = type;
+
+ memcpy(fctx->enc.encr_key, key, key_len);
+
+fc_success:
+ *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags);
+
+success:
+ cpt_ctx->enc_cipher = type;
+
+ return 0;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp(sg_comp_t *list,
+ uint32_t i,
+ phys_addr_t dma_addr,
+ uint32_t size)
+{
+ sg_comp_t *to = &list[i>>2];
+
+ to->u.s.len[i%4] = rte_cpu_to_be_16(size);
+ to->ptr[i%4] = rte_cpu_to_be_64(dma_addr);
+ i++;
+ return i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp_from_buf(sg_comp_t *list,
+ uint32_t i,
+ buf_ptr_t *from)
+{
+ sg_comp_t *to = &list[i>>2];
+
+ to->u.s.len[i%4] = rte_cpu_to_be_16(from->size);
+ to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr);
+ i++;
+ return i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp_from_buf_min(sg_comp_t *list,
+ uint32_t i,
+ buf_ptr_t *from,
+ uint32_t *psize)
+{
+ sg_comp_t *to = &list[i >> 2];
+ uint32_t size = *psize;
+ uint32_t e_len;
+
+ e_len = (size > from->size) ? from->size : size;
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr);
+ *psize -= e_len;
+ i++;
+ return i;
+}
+
+/*
+ * This fills the MC expected SGIO list
+ * from IOV given by user.
+ */
+static __rte_always_inline uint32_t
+fill_sg_comp_from_iov(sg_comp_t *list,
+ uint32_t i,
+ iov_ptr_t *from, uint32_t from_offset,
+ uint32_t *psize, buf_ptr_t *extra_buf,
+ uint32_t extra_offset)
+{
+ int32_t j;
+ uint32_t extra_len = extra_buf ? extra_buf->size : 0;
+ uint32_t size = *psize - extra_len;
+ buf_ptr_t *bufs;
+
+ bufs = from->bufs;
+ for (j = 0; (j < from->buf_cnt) && size; j++) {
+ phys_addr_t e_dma_addr;
+ uint32_t e_len;
+ sg_comp_t *to = &list[i >> 2];
+
+ if (!bufs[j].size)
+ continue;
+
+ if (unlikely(from_offset)) {
+ if (from_offset >= bufs[j].size) {
+ from_offset -= bufs[j].size;
+ continue;
+ }
+ e_dma_addr = bufs[j].dma_addr + from_offset;
+ e_len = (size > (bufs[j].size - from_offset)) ?
+ (bufs[j].size - from_offset) : size;
+ from_offset = 0;
+ } else {
+ e_dma_addr = bufs[j].dma_addr;
+ e_len = (size > bufs[j].size) ?
+ bufs[j].size : size;
+ }
+
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr);
+
+ if (extra_len && (e_len >= extra_offset)) {
+ /* Break the data at given offset */
+ uint32_t next_len = e_len - extra_offset;
+ phys_addr_t next_dma = e_dma_addr + extra_offset;
+
+ if (!extra_offset) {
+ i--;
+ } else {
+ e_len = extra_offset;
+ size -= e_len;
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+ }
+
+ /* Insert extra data ptr */
+ if (extra_len) {
+ i++;
+ to = &list[i >> 2];
+ to->u.s.len[i % 4] =
+ rte_cpu_to_be_16(extra_buf->size);
+ to->ptr[i % 4] =
+ rte_cpu_to_be_64(extra_buf->dma_addr);
+
+ /* size already decremented by extra len */
+ }
+
+ /* insert the rest of the data */
+ if (next_len) {
+ i++;
+ to = &list[i >> 2];
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64(next_dma);
+ size -= next_len;
+ }
+ extra_len = 0;
+
+ } else {
+ size -= e_len;
+ }
+ if (extra_offset)
+ extra_offset -= size;
+ i++;
+ }
+
+ *psize = size;
+ return (uint32_t)i;
+}
+
+static __rte_always_inline int
+cpt_digest_gen_prep(uint32_t flags,
+ uint64_t d_lens,
+ digest_params_t *params,
+ void *op,
+ void **prep_req)
+{
+ struct cpt_request_info *req;
+ uint32_t size, i;
+ int32_t m_size;
+ uint16_t data_len, mac_len, key_len;
+ auth_type_t hash_type;
+ buf_ptr_t *meta_p;
+ struct cpt_ctx *ctx;
+ sg_comp_t *gather_comp;
+ sg_comp_t *scatter_comp;
+ uint8_t *in_buffer;
+ uint32_t g_size_bytes, s_size_bytes;
+ uint64_t dptr_dma, rptr_dma;
+ vq_cmd_word0_t vq_cmd_w0;
+ vq_cmd_word3_t vq_cmd_w3;
+ void *c_vaddr, *m_vaddr;
+ uint64_t c_dma, m_dma;
+ opcode_info_t opcode;
+
+ if (!params || !params->ctx_buf.vaddr)
+ return ERR_BAD_INPUT_ARG;
+
+ ctx = params->ctx_buf.vaddr;
+ meta_p = &params->meta_buf;
+
+ if (!meta_p->vaddr || !meta_p->dma_addr)
+ return ERR_BAD_INPUT_ARG;
+
+ if (meta_p->size < sizeof(struct cpt_request_info))
+ return ERR_BAD_INPUT_ARG;
+
+ m_vaddr = meta_p->vaddr;
+ m_dma = meta_p->dma_addr;
+ m_size = meta_p->size;
+
+ /*
+ * Save initial space that followed app data for completion code &
+ * alternate completion code to fall in same cache line as app data
+ */
+ m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+ m_dma += COMPLETION_CODE_SIZE;
+ size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+ (uint8_t *)m_vaddr;
+ c_vaddr = (uint8_t *)m_vaddr + size;
+ c_dma = m_dma + size;
+ size += sizeof(cpt_res_s_t);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ req = m_vaddr;
+
+ size = sizeof(struct cpt_request_info);
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ hash_type = ctx->hash_type;
+ mac_len = ctx->mac_len;
+ key_len = ctx->auth_key_len;
+ data_len = AUTH_DLEN(d_lens);
+
+ /*GP op header */
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.param2 = rte_cpu_to_be_16(((uint16_t)hash_type << 8));
+ if (ctx->hmac) {
+ opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE;
+ vq_cmd_w0.s.param1 = rte_cpu_to_be_16(key_len);
+ vq_cmd_w0.s.dlen =
+ rte_cpu_to_be_16((data_len + ROUNDUP8(key_len)));
+ } else {
+ opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE;
+ vq_cmd_w0.s.param1 = 0;
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(data_len);
+ }
+
+ opcode.s.minor = 0;
+
+ /* Null auth only case enters the if */
+ if (unlikely(!hash_type && !ctx->enc_cipher)) {
+ opcode.s.major = CPT_MAJOR_OP_MISC;
+ /* Minor op is passthrough */
+ opcode.s.minor = 0x03;
+ /* Send out completion code only */
+ vq_cmd_w0.s.param2 = 0x1;
+ }
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ /* DPTR has SG list */
+ in_buffer = m_vaddr;
+ dptr_dma = m_dma;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* TODO Add error check if space will be sufficient */
+ gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+ /*
+ * Input gather list
+ */
+
+ i = 0;
+
+ if (ctx->hmac) {
+ uint64_t k_dma = params->ctx_buf.dma_addr +
+ offsetof(struct cpt_ctx, auth_key);
+ /* Key */
+ i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len));
+ }
+
+ /* input data */
+ size = data_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov,
+ 0, &size, NULL, 0);
+ if (size) {
+ CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short"
+ " by %dB", size);
+ return ERR_BAD_INPUT_ARG;
+ }
+ } else {
+ /*
+ * Looks like we need to support zero data
+ * gather ptr in case of hash & hmac
+ */
+ i++;
+ }
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+ g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ /*
+ * Output Gather list
+ */
+
+ i = 0;
+ scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+ if (flags & VALID_MAC_BUF) {
+ if (params->mac_buf.size < mac_len)
+ return ERR_BAD_INPUT_ARG;
+
+ size = mac_len;
+ i = fill_sg_comp_from_buf_min(scatter_comp, i,
+ &params->mac_buf, &size);
+ } else {
+ size = mac_len;
+ i = fill_sg_comp_from_iov(scatter_comp, i,
+ params->src_iov, data_len,
+ &size, NULL, 0);
+ if (size) {
+ CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short by"
+ " %dB", size);
+ return ERR_BAD_INPUT_ARG;
+ }
+ }
+
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+ s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+ /* This is DPTR len incase of SG mode */
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* cpt alternate completion address saved earlier */
+ req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+ *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+ rptr_dma = c_dma - 8;
+
+ req->ist.ei1 = dptr_dma;
+ req->ist.ei2 = rptr_dma;
+ /* First 16-bit swap then 64-bit swap */
+ /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+ * to eliminate all the swapping
+ */
+ vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+ /* vq command w3 */
+ vq_cmd_w3.u64 = 0;
+
+ /* 16 byte aligned cpt res address */
+ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+ *req->completion_addr = COMPLETION_CODE_INIT;
+ req->comp_baddr = c_dma;
+
+ /* Fill microcode part of instruction */
+ req->ist.ei0 = vq_cmd_w0.u64;
+ req->ist.ei3 = vq_cmd_w3.u64;
+
+ req->op = op;
+
+ *prep_req = req;
+ return 0;
+}
+
+static __rte_always_inline int
+cpt_enc_hmac_prep(uint32_t flags,
+ uint64_t d_offs,
+ uint64_t d_lens,
+ fc_params_t *fc_params,
+ void *op,
+ void **prep_req)
+{
+ uint32_t iv_offset = 0;
+ int32_t inputlen, outputlen, enc_dlen, auth_dlen;
+ struct cpt_ctx *cpt_ctx;
+ uint32_t cipher_type, hash_type;
+ uint32_t mac_len, size;
+ uint8_t iv_len = 16;
+ struct cpt_request_info *req;
+ buf_ptr_t *meta_p, *aad_buf = NULL;
+ uint32_t encr_offset, auth_offset;
+ uint32_t encr_data_len, auth_data_len, aad_len = 0;
+ uint32_t passthrough_len = 0;
+ void *m_vaddr, *offset_vaddr;
+ uint64_t m_dma, offset_dma, ctx_dma;
+ vq_cmd_word0_t vq_cmd_w0;
+ vq_cmd_word3_t vq_cmd_w3;
+ void *c_vaddr;
+ uint64_t c_dma;
+ int32_t m_size;
+ opcode_info_t opcode;
+
+ meta_p = &fc_params->meta_buf;
+ m_vaddr = meta_p->vaddr;
+ m_dma = meta_p->dma_addr;
+ m_size = meta_p->size;
+
+ encr_offset = ENCR_OFFSET(d_offs);
+ auth_offset = AUTH_OFFSET(d_offs);
+ encr_data_len = ENCR_DLEN(d_lens);
+ auth_data_len = AUTH_DLEN(d_lens);
+ if (unlikely(flags & VALID_AAD_BUF)) {
+ /*
+ * We dont support both aad
+ * and auth data separately
+ */
+ auth_data_len = 0;
+ auth_offset = 0;
+ aad_len = fc_params->aad_buf.size;
+ aad_buf = &fc_params->aad_buf;
+ }
+ cpt_ctx = fc_params->ctx_buf.vaddr;
+ cipher_type = cpt_ctx->enc_cipher;
+ hash_type = cpt_ctx->hash_type;
+ mac_len = cpt_ctx->mac_len;
+
+ /*
+ * Save initial space that followed app data for completion code &
+ * alternate completion code to fall in same cache line as app data
+ */
+ m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+ m_dma += COMPLETION_CODE_SIZE;
+ size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+ (uint8_t *)m_vaddr;
+
+ c_vaddr = (uint8_t *)m_vaddr + size;
+ c_dma = m_dma + size;
+ size += sizeof(cpt_res_s_t);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* start cpt request info struct at 8 byte boundary */
+ size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
+ (uint8_t *)m_vaddr;
+
+ req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
+
+ size += sizeof(struct cpt_request_info);
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ if (hash_type == GMAC_TYPE)
+ encr_data_len = 0;
+
+ if (unlikely(!(flags & VALID_IV_BUF))) {
+ iv_len = 0;
+ iv_offset = ENCR_IV_OFFSET(d_offs);
+ }
+
+ if (unlikely(flags & VALID_AAD_BUF)) {
+ /*
+ * When AAD is given, data above encr_offset is pass through
+ * Since AAD is given as separate pointer and not as offset,
+ * this is a special case as we need to fragment input data
+ * into passthrough + encr_data and then insert AAD in between.
+ */
+ if (hash_type != GMAC_TYPE) {
+ passthrough_len = encr_offset;
+ auth_offset = passthrough_len + iv_len;
+ encr_offset = passthrough_len + aad_len + iv_len;
+ auth_data_len = aad_len + encr_data_len;
+ } else {
+ passthrough_len = 16 + aad_len;
+ auth_offset = passthrough_len + iv_len;
+ auth_data_len = aad_len;
+ }
+ } else {
+ encr_offset += iv_len;
+ auth_offset += iv_len;
+ }
+
+ /* Encryption */
+ opcode.s.major = CPT_MAJOR_OP_FC;
+ opcode.s.minor = 0;
+
+ auth_dlen = auth_offset + auth_data_len;
+ enc_dlen = encr_data_len + encr_offset;
+ if (unlikely(encr_data_len & 0xf)) {
+ if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB))
+ enc_dlen = ROUNDUP8(encr_data_len) + encr_offset;
+ else if (likely((cipher_type == AES_CBC) ||
+ (cipher_type == AES_ECB)))
+ enc_dlen = ROUNDUP16(encr_data_len) + encr_offset;
+ }
+
+ if (unlikely(hash_type == GMAC_TYPE)) {
+ encr_offset = auth_dlen;
+ enc_dlen = 0;
+ }
+
+ if (unlikely(auth_dlen > enc_dlen)) {
+ inputlen = auth_dlen;
+ outputlen = auth_dlen + mac_len;
+ } else {
+ inputlen = enc_dlen;
+ outputlen = enc_dlen + mac_len;
+ }
+
+ /* GP op header */
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+ vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len);
+ /*
+ * In 83XX since we have a limitation of
+ * IV & Offset control word not part of instruction
+ * and need to be part of Data Buffer, we check if
+ * head room is there and then only do the Direct mode processing
+ */
+ if (likely((flags & SINGLE_BUF_INPLACE) &&
+ (flags & SINGLE_BUF_HEADTAILROOM))) {
+ void *dm_vaddr = fc_params->bufs[0].vaddr;
+ uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
+ /*
+ * This flag indicates that there is 24 bytes head room and
+ * 8 bytes tail room available, so that we get to do
+ * DIRECT MODE with limitation
+ */
+
+ offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
+ offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
+
+ /* DPTR */
+ req->ist.ei1 = offset_dma;
+ /* RPTR should just exclude offset control word */
+ req->ist.ei2 = dm_dma_addr - iv_len;
+ req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
+ + outputlen - iv_len);
+
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN);
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ if (likely(iv_len)) {
+ uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
+ + OFF_CTRL_LEN);
+ uint64_t *src = fc_params->iv_buf;
+ dest[0] = src[0];
+ dest[1] = src[1];
+ }
+
+ *(uint64_t *)offset_vaddr =
+ rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
+ ((uint64_t)iv_offset << 8) |
+ ((uint64_t)auth_offset));
+
+ } else {
+ uint32_t i, g_size_bytes, s_size_bytes;
+ uint64_t dptr_dma, rptr_dma;
+ sg_comp_t *gather_comp;
+ sg_comp_t *scatter_comp;
+ uint8_t *in_buffer;
+
+ /* This falls under strict SG mode */
+ offset_vaddr = m_vaddr;
+ offset_dma = m_dma;
+ size = OFF_CTRL_LEN + iv_len;
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ opcode.s.major |= CPT_DMA_MODE;
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ if (likely(iv_len)) {
+ uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
+ + OFF_CTRL_LEN);
+ uint64_t *src = fc_params->iv_buf;
+ dest[0] = src[0];
+ dest[1] = src[1];
+ }
+
+ *(uint64_t *)offset_vaddr =
+ rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
+ ((uint64_t)iv_offset << 8) |
+ ((uint64_t)auth_offset));
+
+ /* DPTR has SG list */
+ in_buffer = m_vaddr;
+ dptr_dma = m_dma;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* TODO Add error check if space will be sufficient */
+ gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+ /*
+ * Input Gather List
+ */
+
+ i = 0;
+
+ /* Offset control word that includes iv */
+ i = fill_sg_comp(gather_comp, i, offset_dma,
+ OFF_CTRL_LEN + iv_len);
+
+ /* Add input data */
+ size = inputlen - iv_len;
+ if (likely(size)) {
+ uint32_t aad_offset = aad_len ? passthrough_len : 0;
+
+ if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+ i = fill_sg_comp_from_buf_min(gather_comp, i,
+ fc_params->bufs,
+ &size);
+ } else {
+ i = fill_sg_comp_from_iov(gather_comp, i,
+ fc_params->src_iov,
+ 0, &size,
+ aad_buf, aad_offset);
+ }
+
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return ERR_BAD_INPUT_ARG;
+ }
+ }
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+ g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ /*
+ * Output Scatter list
+ */
+ i = 0;
+ scatter_comp =
+ (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+ /* Add IV */
+ if (likely(iv_len)) {
+ i = fill_sg_comp(scatter_comp, i,
+ offset_dma + OFF_CTRL_LEN,
+ iv_len);
+ }
+
+ /* output data or output data + digest*/
+ if (unlikely(flags & VALID_MAC_BUF)) {
+ size = outputlen - iv_len - mac_len;
+ if (size) {
+ uint32_t aad_offset =
+ aad_len ? passthrough_len : 0;
+
+ if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+ i = fill_sg_comp_from_buf_min(
+ scatter_comp,
+ i,
+ fc_params->bufs,
+ &size);
+ } else {
+ i = fill_sg_comp_from_iov(scatter_comp,
+ i,
+ fc_params->dst_iov,
+ 0,
+ &size,
+ aad_buf,
+ aad_offset);
+ }
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ /* mac_data */
+ if (mac_len) {
+ i = fill_sg_comp_from_buf(scatter_comp, i,
+ &fc_params->mac_buf);
+ }
+ } else {
+ /* Output including mac */
+ size = outputlen - iv_len;
+ if (likely(size)) {
+ uint32_t aad_offset =
+ aad_len ? passthrough_len : 0;
+
+ if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+ i = fill_sg_comp_from_buf_min(
+ scatter_comp,
+ i,
+ fc_params->bufs,
+ &size);
+ } else {
+ i = fill_sg_comp_from_iov(scatter_comp,
+ i,
+ fc_params->dst_iov,
+ 0,
+ &size,
+ aad_buf,
+ aad_offset);
+ }
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer"
+ " space, size %d needed",
+ size);
+ return ERR_BAD_INPUT_ARG;
+ }
+ }
+ }
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+ s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+ /* This is DPTR len incase of SG mode */
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* cpt alternate completion address saved earlier */
+ req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+ *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+ rptr_dma = c_dma - 8;
+
+ req->ist.ei1 = dptr_dma;
+ req->ist.ei2 = rptr_dma;
+ }
+
+ /* First 16-bit swap then 64-bit swap */
+ /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+ * to eliminate all the swapping
+ */
+ vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+ ctx_dma = fc_params->ctx_buf.dma_addr +
+ offsetof(struct cpt_ctx, fctx);
+ /* vq command w3 */
+ vq_cmd_w3.u64 = 0;
+ vq_cmd_w3.s.grp = 0;
+ vq_cmd_w3.s.cptr = ctx_dma;
+
+ /* 16 byte aligned cpt res address */
+ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+ *req->completion_addr = COMPLETION_CODE_INIT;
+ req->comp_baddr = c_dma;
+
+ /* Fill microcode part of instruction */
+ req->ist.ei0 = vq_cmd_w0.u64;
+ req->ist.ei3 = vq_cmd_w3.u64;
+
+ req->op = op;
+
+ *prep_req = req;
+ return 0;
+}
+
+static __rte_always_inline int
+cpt_dec_hmac_prep(uint32_t flags,
+ uint64_t d_offs,
+ uint64_t d_lens,
+ fc_params_t *fc_params,
+ void *op,
+ void **prep_req)
+{
+ uint32_t iv_offset = 0, size;
+ int32_t inputlen, outputlen, enc_dlen, auth_dlen;
+ struct cpt_ctx *cpt_ctx;
+ int32_t hash_type, mac_len, m_size;
+ uint8_t iv_len = 16;
+ struct cpt_request_info *req;
+ buf_ptr_t *meta_p, *aad_buf = NULL;
+ uint32_t encr_offset, auth_offset;
+ uint32_t encr_data_len, auth_data_len, aad_len = 0;
+ uint32_t passthrough_len = 0;
+ void *m_vaddr, *offset_vaddr;
+ uint64_t m_dma, offset_dma, ctx_dma;
+ opcode_info_t opcode;
+ vq_cmd_word0_t vq_cmd_w0;
+ vq_cmd_word3_t vq_cmd_w3;
+ void *c_vaddr;
+ uint64_t c_dma;
+
+ meta_p = &fc_params->meta_buf;
+ m_vaddr = meta_p->vaddr;
+ m_dma = meta_p->dma_addr;
+ m_size = meta_p->size;
+
+ encr_offset = ENCR_OFFSET(d_offs);
+ auth_offset = AUTH_OFFSET(d_offs);
+ encr_data_len = ENCR_DLEN(d_lens);
+ auth_data_len = AUTH_DLEN(d_lens);
+
+ if (unlikely(flags & VALID_AAD_BUF)) {
+ /*
+ * We dont support both aad
+ * and auth data separately
+ */
+ auth_data_len = 0;
+ auth_offset = 0;
+ aad_len = fc_params->aad_buf.size;
+ aad_buf = &fc_params->aad_buf;
+ }
+
+ cpt_ctx = fc_params->ctx_buf.vaddr;
+ hash_type = cpt_ctx->hash_type;
+ mac_len = cpt_ctx->mac_len;
+
+ if (hash_type == GMAC_TYPE)
+ encr_data_len = 0;
+
+ if (unlikely(!(flags & VALID_IV_BUF))) {
+ iv_len = 0;
+ iv_offset = ENCR_IV_OFFSET(d_offs);
+ }
+
+ if (unlikely(flags & VALID_AAD_BUF)) {
+ /*
+ * When AAD is given, data above encr_offset is pass through
+ * Since AAD is given as separate pointer and not as offset,
+ * this is a special case as we need to fragment input data
+ * into passthrough + encr_data and then insert AAD in between.
+ */
+ if (hash_type != GMAC_TYPE) {
+ passthrough_len = encr_offset;
+ auth_offset = passthrough_len + iv_len;
+ encr_offset = passthrough_len + aad_len + iv_len;
+ auth_data_len = aad_len + encr_data_len;
+ } else {
+ passthrough_len = 16 + aad_len;
+ auth_offset = passthrough_len + iv_len;
+ auth_data_len = aad_len;
+ }
+ } else {
+ encr_offset += iv_len;
+ auth_offset += iv_len;
+ }
+
+ /*
+ * Save initial space that followed app data for completion code &
+ * alternate completion code to fall in same cache line as app data
+ */
+ m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+ m_dma += COMPLETION_CODE_SIZE;
+ size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+ (uint8_t *)m_vaddr;
+ c_vaddr = (uint8_t *)m_vaddr + size;
+ c_dma = m_dma + size;
+ size += sizeof(cpt_res_s_t);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* start cpt request info structure at 8 byte alignment */
+ size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
+ (uint8_t *)m_vaddr;
+
+ req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
+
+ size += sizeof(struct cpt_request_info);
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* Decryption */
+ opcode.s.major = CPT_MAJOR_OP_FC;
+ opcode.s.minor = 1;
+
+ enc_dlen = encr_offset + encr_data_len;
+ auth_dlen = auth_offset + auth_data_len;
+
+ if (auth_dlen > enc_dlen) {
+ inputlen = auth_dlen + mac_len;
+ outputlen = auth_dlen;
+ } else {
+ inputlen = enc_dlen + mac_len;
+ outputlen = enc_dlen;
+ }
+
+ if (hash_type == GMAC_TYPE)
+ encr_offset = inputlen;
+
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+ vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len);
+
+ /*
+ * In 83XX since we have a limitation of
+ * IV & Offset control word not part of instruction
+ * and need to be part of Data Buffer, we check if
+ * head room is there and then only do the Direct mode processing
+ */
+ if (likely((flags & SINGLE_BUF_INPLACE) &&
+ (flags & SINGLE_BUF_HEADTAILROOM))) {
+ void *dm_vaddr = fc_params->bufs[0].vaddr;
+ uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
+ /*
+ * This flag indicates that there is 24 bytes head room and
+ * 8 bytes tail room available, so that we get to do
+ * DIRECT MODE with limitation
+ */
+
+ offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
+ offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
+ req->ist.ei1 = offset_dma;
+
+ /* RPTR should just exclude offset control word */
+ req->ist.ei2 = dm_dma_addr - iv_len;
+
+ req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr +
+ outputlen - iv_len);
+ /* since this is decryption,
+ * don't touch the content of
+ * alternate ccode space as it contains
+ * hmac.
+ */
+
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN);
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ if (likely(iv_len)) {
+ uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
+ OFF_CTRL_LEN);
+ uint64_t *src = fc_params->iv_buf;
+ dest[0] = src[0];
+ dest[1] = src[1];
+ }
+
+ *(uint64_t *)offset_vaddr =
+ rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
+ ((uint64_t)iv_offset << 8) |
+ ((uint64_t)auth_offset));
+
+ } else {
+ uint64_t dptr_dma, rptr_dma;
+ uint32_t g_size_bytes, s_size_bytes;
+ sg_comp_t *gather_comp;
+ sg_comp_t *scatter_comp;
+ uint8_t *in_buffer;
+ uint8_t i = 0;
+
+ /* This falls under strict SG mode */
+ offset_vaddr = m_vaddr;
+ offset_dma = m_dma;
+ size = OFF_CTRL_LEN + iv_len;
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ opcode.s.major |= CPT_DMA_MODE;
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ if (likely(iv_len)) {
+ uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
+ OFF_CTRL_LEN);
+ uint64_t *src = fc_params->iv_buf;
+ dest[0] = src[0];
+ dest[1] = src[1];
+ }
+
+ *(uint64_t *)offset_vaddr =
+ rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
+ ((uint64_t)iv_offset << 8) |
+ ((uint64_t)auth_offset));
+
+ /* DPTR has SG list */
+ in_buffer = m_vaddr;
+ dptr_dma = m_dma;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* TODO Add error check if space will be sufficient */
+ gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+ /*
+ * Input Gather List
+ */
+ i = 0;
+
+ /* Offset control word that includes iv */
+ i = fill_sg_comp(gather_comp, i, offset_dma,
+ OFF_CTRL_LEN + iv_len);
+
+ /* Add input data */
+ if (flags & VALID_MAC_BUF) {
+ size = inputlen - iv_len - mac_len;
+ if (size) {
+ /* input data only */
+ if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+ i = fill_sg_comp_from_buf_min(
+ gather_comp, i,
+ fc_params->bufs,
+ &size);
+ } else {
+ uint32_t aad_offset = aad_len ?
+ passthrough_len : 0;
+
+ i = fill_sg_comp_from_iov(gather_comp,
+ i,
+ fc_params->src_iov,
+ 0, &size,
+ aad_buf,
+ aad_offset);
+ }
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+
+ /* mac data */
+ if (mac_len) {
+ i = fill_sg_comp_from_buf(gather_comp, i,
+ &fc_params->mac_buf);
+ }
+ } else {
+ /* input data + mac */
+ size = inputlen - iv_len;
+ if (size) {
+ if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+ i = fill_sg_comp_from_buf_min(
+ gather_comp, i,
+ fc_params->bufs,
+ &size);
+ } else {
+ uint32_t aad_offset = aad_len ?
+ passthrough_len : 0;
+
+ if (!fc_params->src_iov)
+ return ERR_BAD_INPUT_ARG;
+
+ i = fill_sg_comp_from_iov(
+ gather_comp, i,
+ fc_params->src_iov,
+ 0, &size,
+ aad_buf,
+ aad_offset);
+ }
+
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ }
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+ g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ /*
+ * Output Scatter List
+ */
+
+ i = 0;
+ scatter_comp =
+ (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+ /* Add iv */
+ if (iv_len) {
+ i = fill_sg_comp(scatter_comp, i,
+ offset_dma + OFF_CTRL_LEN,
+ iv_len);
+ }
+
+ /* Add output data */
+ size = outputlen - iv_len;
+ if (size) {
+ if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+ /* handle single buffer here */
+ i = fill_sg_comp_from_buf_min(scatter_comp, i,
+ fc_params->bufs,
+ &size);
+ } else {
+ uint32_t aad_offset = aad_len ?
+ passthrough_len : 0;
+
+ if (!fc_params->dst_iov)
+ return ERR_BAD_INPUT_ARG;
+
+ i = fill_sg_comp_from_iov(scatter_comp, i,
+ fc_params->dst_iov, 0,
+ &size, aad_buf,
+ aad_offset);
+ }
+
+ if (unlikely(size))
+ return ERR_BAD_INPUT_ARG;
+ }
+
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+ s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+ /* This is DPTR len incase of SG mode */
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* cpt alternate completion address saved earlier */
+ req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+ *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+ rptr_dma = c_dma - 8;
+ size += COMPLETION_CODE_SIZE;
+
+ req->ist.ei1 = dptr_dma;
+ req->ist.ei2 = rptr_dma;
+ }
+
+ /* First 16-bit swap then 64-bit swap */
+ /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+ * to eliminate all the swapping
+ */
+ vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+ ctx_dma = fc_params->ctx_buf.dma_addr +
+ offsetof(struct cpt_ctx, fctx);
+ /* vq command w3 */
+ vq_cmd_w3.u64 = 0;
+ vq_cmd_w3.s.grp = 0;
+ vq_cmd_w3.s.cptr = ctx_dma;
+
+ /* 16 byte aligned cpt res address */
+ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+ *req->completion_addr = COMPLETION_CODE_INIT;
+ req->comp_baddr = c_dma;
+
+ /* Fill microcode part of instruction */
+ req->ist.ei0 = vq_cmd_w0.u64;
+ req->ist.ei3 = vq_cmd_w3.u64;
+
+ req->op = op;
+
+ *prep_req = req;
+ return 0;
+}
+
+static __rte_always_inline int
+cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
+ uint64_t d_offs,
+ uint64_t d_lens,
+ fc_params_t *params,
+ void *op,
+ void **prep_req)
+{
+ uint32_t size;
+ int32_t inputlen, outputlen;
+ struct cpt_ctx *cpt_ctx;
+ uint32_t mac_len = 0;
+ uint8_t snow3g, j;
+ struct cpt_request_info *req;
+ buf_ptr_t *buf_p;
+ uint32_t encr_offset = 0, auth_offset = 0;
+ uint32_t encr_data_len = 0, auth_data_len = 0;
+ int flags, iv_len = 16, m_size;
+ void *m_vaddr, *c_vaddr;
+ uint64_t m_dma, c_dma, offset_ctrl;
+ uint64_t *offset_vaddr, offset_dma;
+ uint32_t *iv_s, iv[4];
+ vq_cmd_word0_t vq_cmd_w0;
+ vq_cmd_word3_t vq_cmd_w3;
+ opcode_info_t opcode;
+
+ buf_p = &params->meta_buf;
+ m_vaddr = buf_p->vaddr;
+ m_dma = buf_p->dma_addr;
+ m_size = buf_p->size;
+
+ cpt_ctx = params->ctx_buf.vaddr;
+ flags = cpt_ctx->zsk_flags;
+ mac_len = cpt_ctx->mac_len;
+ snow3g = cpt_ctx->snow3g;
+
+ /*
+ * Save initial space that followed app data for completion code &
+ * alternate completion code to fall in same cache line as app data
+ */
+ m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+ m_dma += COMPLETION_CODE_SIZE;
+ size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+ (uint8_t *)m_vaddr;
+
+ c_vaddr = (uint8_t *)m_vaddr + size;
+ c_dma = m_dma + size;
+ size += sizeof(cpt_res_s_t);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* Reserve memory for cpt request info */
+ req = m_vaddr;
+
+ size = sizeof(struct cpt_request_info);
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G;
+
+ /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
+ opcode.s.minor = ((1 << 6) | (snow3g << 5) | (0 << 4) |
+ (0 << 3) | (flags & 0x7));
+
+ if (flags == 0x1) {
+ /*
+ * Microcode expects offsets in bytes
+ * TODO: Rounding off
+ */
+ auth_data_len = AUTH_DLEN(d_lens);
+
+ /* EIA3 or UIA2 */
+ auth_offset = AUTH_OFFSET(d_offs);
+ auth_offset = auth_offset / 8;
+
+ /* consider iv len */
+ auth_offset += iv_len;
+
+ inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
+ outputlen = mac_len;
+
+ offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset);
+
+ } else {
+ /* EEA3 or UEA2 */
+ /*
+ * Microcode expects offsets in bytes
+ * TODO: Rounding off
+ */
+ encr_data_len = ENCR_DLEN(d_lens);
+
+ encr_offset = ENCR_OFFSET(d_offs);
+ encr_offset = encr_offset / 8;
+ /* consider iv len */
+ encr_offset += iv_len;
+
+ inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
+ outputlen = inputlen;
+
+ /* iv offset is 0 */
+ offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+ }
+
+ /* IV */
+ iv_s = (flags == 0x1) ? params->auth_iv_buf :
+ params->iv_buf;
+
+ if (snow3g) {
+ /*
+ * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
+ * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
+ */
+
+ for (j = 0; j < 4; j++)
+ iv[j] = iv_s[3 - j];
+ } else {
+ /* ZUC doesn't need a swap */
+ for (j = 0; j < 4; j++)
+ iv[j] = iv_s[j];
+ }
+
+ /*
+ * GP op header, lengths are expected in bits.
+ */
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+ vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len);
+
+ /*
+ * In 83XX since we have a limitation of
+ * IV & Offset control word not part of instruction
+ * and need to be part of Data Buffer, we check if
+ * head room is there and then only do the Direct mode processing
+ */
+ if (likely((req_flags & SINGLE_BUF_INPLACE) &&
+ (req_flags & SINGLE_BUF_HEADTAILROOM))) {
+ void *dm_vaddr = params->bufs[0].vaddr;
+ uint64_t dm_dma_addr = params->bufs[0].dma_addr;
+ /*
+ * This flag indicates that there is 24 bytes head room and
+ * 8 bytes tail room available, so that we get to do
+ * DIRECT MODE with limitation
+ */
+
+ offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
+ OFF_CTRL_LEN - iv_len);
+ offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
+
+ /* DPTR */
+ req->ist.ei1 = offset_dma;
+ /* RPTR should just exclude offset control word */
+ req->ist.ei2 = dm_dma_addr - iv_len;
+ req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
+ + outputlen - iv_len);
+
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN);
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ if (likely(iv_len)) {
+ uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
+ + OFF_CTRL_LEN);
+ memcpy(iv_d, iv, 16);
+ }
+
+ *offset_vaddr = offset_ctrl;
+ } else {
+ uint32_t i, g_size_bytes, s_size_bytes;
+ uint64_t dptr_dma, rptr_dma;
+ sg_comp_t *gather_comp;
+ sg_comp_t *scatter_comp;
+ uint8_t *in_buffer;
+ uint32_t *iv_d;
+
+ /* save space for iv */
+ offset_vaddr = m_vaddr;
+ offset_dma = m_dma;
+
+ m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
+ m_dma += OFF_CTRL_LEN + iv_len;
+ m_size -= OFF_CTRL_LEN + iv_len;
+
+ opcode.s.major |= CPT_DMA_MODE;
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ /* DPTR has SG list */
+ in_buffer = m_vaddr;
+ dptr_dma = m_dma;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* TODO Add error check if space will be sufficient */
+ gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+ /*
+ * Input Gather List
+ */
+ i = 0;
+
+ /* Offset control word followed by iv */
+
+ i = fill_sg_comp(gather_comp, i, offset_dma,
+ OFF_CTRL_LEN + iv_len);
+
+ /* iv offset is 0 */
+ *offset_vaddr = offset_ctrl;
+
+ iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
+ memcpy(iv_d, iv, 16);
+
+ /* input data */
+ size = inputlen - iv_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(gather_comp, i,
+ params->src_iov,
+ 0, &size, NULL, 0);
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+ g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ /*
+ * Output Scatter List
+ */
+
+ i = 0;
+ scatter_comp =
+ (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+ if (flags == 0x1) {
+ /* IV in SLIST only for EEA3 & UEA2 */
+ iv_len = 0;
+ }
+
+ if (iv_len) {
+ i = fill_sg_comp(scatter_comp, i,
+ offset_dma + OFF_CTRL_LEN, iv_len);
+ }
+
+ /* Add output data */
+ if (req_flags & VALID_MAC_BUF) {
+ size = outputlen - iv_len - mac_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(scatter_comp, i,
+ params->dst_iov, 0,
+ &size, NULL, 0);
+
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+
+ /* mac data */
+ if (mac_len) {
+ i = fill_sg_comp_from_buf(scatter_comp, i,
+ &params->mac_buf);
+ }
+ } else {
+ /* Output including mac */
+ size = outputlen - iv_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(scatter_comp, i,
+ params->dst_iov, 0,
+ &size, NULL, 0);
+
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ }
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+ s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+ /* This is DPTR len incase of SG mode */
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* cpt alternate completion address saved earlier */
+ req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+ *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+ rptr_dma = c_dma - 8;
+
+ req->ist.ei1 = dptr_dma;
+ req->ist.ei2 = rptr_dma;
+ }
+
+ /* First 16-bit swap then 64-bit swap */
+ /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+ * to eliminate all the swapping
+ */
+ vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+ /* vq command w3 */
+ vq_cmd_w3.u64 = 0;
+ vq_cmd_w3.s.grp = 0;
+ vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
+ offsetof(struct cpt_ctx, zs_ctx);
+
+ /* 16 byte aligned cpt res address */
+ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+ *req->completion_addr = COMPLETION_CODE_INIT;
+ req->comp_baddr = c_dma;
+
+ /* Fill microcode part of instruction */
+ req->ist.ei0 = vq_cmd_w0.u64;
+ req->ist.ei3 = vq_cmd_w3.u64;
+
+ req->op = op;
+
+ *prep_req = req;
+ return 0;
+}
+
+static __rte_always_inline int
+cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
+ uint64_t d_offs,
+ uint64_t d_lens,
+ fc_params_t *params,
+ void *op,
+ void **prep_req)
+{
+ uint32_t size;
+ int32_t inputlen = 0, outputlen;
+ struct cpt_ctx *cpt_ctx;
+ uint8_t snow3g, iv_len = 16;
+ struct cpt_request_info *req;
+ buf_ptr_t *buf_p;
+ uint32_t encr_offset;
+ uint32_t encr_data_len;
+ int flags, m_size;
+ void *m_vaddr, *c_vaddr;
+ uint64_t m_dma, c_dma;
+ uint64_t *offset_vaddr, offset_dma;
+ uint32_t *iv_s, iv[4], j;
+ vq_cmd_word0_t vq_cmd_w0;
+ vq_cmd_word3_t vq_cmd_w3;
+ opcode_info_t opcode;
+
+ buf_p = &params->meta_buf;
+ m_vaddr = buf_p->vaddr;
+ m_dma = buf_p->dma_addr;
+ m_size = buf_p->size;
+
+ /*
+ * Microcode expects offsets in bytes
+ * TODO: Rounding off
+ */
+ encr_offset = ENCR_OFFSET(d_offs) / 8;
+ encr_data_len = ENCR_DLEN(d_lens);
+
+ cpt_ctx = params->ctx_buf.vaddr;
+ flags = cpt_ctx->zsk_flags;
+ snow3g = cpt_ctx->snow3g;
+ /*
+ * Save initial space that followed app data for completion code &
+ * alternate completion code to fall in same cache line as app data
+ */
+ m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+ m_dma += COMPLETION_CODE_SIZE;
+ size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+ (uint8_t *)m_vaddr;
+
+ c_vaddr = (uint8_t *)m_vaddr + size;
+ c_dma = m_dma + size;
+ size += sizeof(cpt_res_s_t);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* Reserve memory for cpt request info */
+ req = m_vaddr;
+
+ size = sizeof(struct cpt_request_info);
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G;
+
+ /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
+ opcode.s.minor = ((1 << 6) | (snow3g << 5) | (0 << 4) |
+ (0 << 3) | (flags & 0x7));
+
+ /* consider iv len */
+ encr_offset += iv_len;
+
+ inputlen = encr_offset +
+ (RTE_ALIGN(encr_data_len, 8) / 8);
+ outputlen = inputlen;
+
+ /* IV */
+ iv_s = params->iv_buf;
+ if (snow3g) {
+ /*
+ * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
+ * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
+ */
+
+ for (j = 0; j < 4; j++)
+ iv[j] = iv_s[3 - j];
+ } else {
+ /* ZUC doesn't need a swap */
+ for (j = 0; j < 4; j++)
+ iv[j] = iv_s[j];
+ }
+
+ /*
+ * GP op header, lengths are expected in bits.
+ */
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+
+ /*
+ * In 83XX since we have a limitation of
+ * IV & Offset control word not part of instruction
+ * and need to be part of Data Buffer, we check if
+ * head room is there and then only do the Direct mode processing
+ */
+ if (likely((req_flags & SINGLE_BUF_INPLACE) &&
+ (req_flags & SINGLE_BUF_HEADTAILROOM))) {
+ void *dm_vaddr = params->bufs[0].vaddr;
+ uint64_t dm_dma_addr = params->bufs[0].dma_addr;
+ /*
+ * This flag indicates that there is 24 bytes head room and
+ * 8 bytes tail room available, so that we get to do
+ * DIRECT MODE with limitation
+ */
+
+ offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
+ OFF_CTRL_LEN - iv_len);
+ offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
+
+ /* DPTR */
+ req->ist.ei1 = offset_dma;
+ /* RPTR should just exclude offset control word */
+ req->ist.ei2 = dm_dma_addr - iv_len;
+ req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
+ + outputlen - iv_len);
+
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN);
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ if (likely(iv_len)) {
+ uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
+ + OFF_CTRL_LEN);
+ memcpy(iv_d, iv, 16);
+ }
+
+ /* iv offset is 0 */
+ *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+ } else {
+ uint32_t i, g_size_bytes, s_size_bytes;
+ uint64_t dptr_dma, rptr_dma;
+ sg_comp_t *gather_comp;
+ sg_comp_t *scatter_comp;
+ uint8_t *in_buffer;
+ uint32_t *iv_d;
+
+ /* save space for offset and iv... */
+ offset_vaddr = m_vaddr;
+ offset_dma = m_dma;
+
+ m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
+ m_dma += OFF_CTRL_LEN + iv_len;
+ m_size -= OFF_CTRL_LEN + iv_len;
+
+ opcode.s.major |= CPT_DMA_MODE;
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ /* DPTR has SG list */
+ in_buffer = m_vaddr;
+ dptr_dma = m_dma;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* TODO Add error check if space will be sufficient */
+ gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+ /*
+ * Input Gather List
+ */
+ i = 0;
+
+ /* Offset control word */
+
+ /* iv offset is 0 */
+ *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+
+ i = fill_sg_comp(gather_comp, i, offset_dma,
+ OFF_CTRL_LEN + iv_len);
+
+ iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
+ memcpy(iv_d, iv, 16);
+
+ /* Add input data */
+ size = inputlen - iv_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(gather_comp, i,
+ params->src_iov,
+ 0, &size, NULL, 0);
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+ g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ /*
+ * Output Scatter List
+ */
+
+ i = 0;
+ scatter_comp =
+ (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+ /* IV */
+ i = fill_sg_comp(scatter_comp, i,
+ offset_dma + OFF_CTRL_LEN,
+ iv_len);
+
+ /* Add output data */
+ size = outputlen - iv_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(scatter_comp, i,
+ params->dst_iov, 0,
+ &size, NULL, 0);
+
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+ s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+ /* This is DPTR len incase of SG mode */
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* cpt alternate completion address saved earlier */
+ req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+ *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+ rptr_dma = c_dma - 8;
+
+ req->ist.ei1 = dptr_dma;
+ req->ist.ei2 = rptr_dma;
+ }
+
+ /* First 16-bit swap then 64-bit swap */
+ /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+ * to eliminate all the swapping
+ */
+ vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+ /* vq command w3 */
+ vq_cmd_w3.u64 = 0;
+ vq_cmd_w3.s.grp = 0;
+ vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
+ offsetof(struct cpt_ctx, zs_ctx);
+
+ /* 16 byte aligned cpt res address */
+ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+ *req->completion_addr = COMPLETION_CODE_INIT;
+ req->comp_baddr = c_dma;
+
+ /* Fill microcode part of instruction */
+ req->ist.ei0 = vq_cmd_w0.u64;
+ req->ist.ei3 = vq_cmd_w3.u64;
+
+ req->op = op;
+
+ *prep_req = req;
+ return 0;
+}
+
+static __rte_always_inline int
+cpt_kasumi_enc_prep(uint32_t req_flags,
+ uint64_t d_offs,
+ uint64_t d_lens,
+ fc_params_t *params,
+ void *op,
+ void **prep_req)
+{
+ uint32_t size;
+ int32_t inputlen = 0, outputlen = 0;
+ struct cpt_ctx *cpt_ctx;
+ uint32_t mac_len = 0;
+ uint8_t i = 0;
+ struct cpt_request_info *req;
+ buf_ptr_t *buf_p;
+ uint32_t encr_offset, auth_offset;
+ uint32_t encr_data_len, auth_data_len;
+ int flags, m_size;
+ uint8_t *iv_s, *iv_d, iv_len = 8;
+ uint8_t dir = 0;
+ void *m_vaddr, *c_vaddr;
+ uint64_t m_dma, c_dma;
+ uint64_t *offset_vaddr, offset_dma;
+ vq_cmd_word0_t vq_cmd_w0;
+ vq_cmd_word3_t vq_cmd_w3;
+ opcode_info_t opcode;
+ uint8_t *in_buffer;
+ uint32_t g_size_bytes, s_size_bytes;
+ uint64_t dptr_dma, rptr_dma;
+ sg_comp_t *gather_comp;
+ sg_comp_t *scatter_comp;
+
+ buf_p = &params->meta_buf;
+ m_vaddr = buf_p->vaddr;
+ m_dma = buf_p->dma_addr;
+ m_size = buf_p->size;
+
+ encr_offset = ENCR_OFFSET(d_offs) / 8;
+ auth_offset = AUTH_OFFSET(d_offs) / 8;
+ encr_data_len = ENCR_DLEN(d_lens);
+ auth_data_len = AUTH_DLEN(d_lens);
+
+ cpt_ctx = params->ctx_buf.vaddr;
+ flags = cpt_ctx->zsk_flags;
+ mac_len = cpt_ctx->mac_len;
+
+ if (flags == 0x0)
+ iv_s = params->iv_buf;
+ else
+ iv_s = params->auth_iv_buf;
+
+ dir = iv_s[8] & 0x1;
+
+ /*
+ * Save initial space that followed app data for completion code &
+ * alternate completion code to fall in same cache line as app data
+ */
+ m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+ m_dma += COMPLETION_CODE_SIZE;
+ size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+ (uint8_t *)m_vaddr;
+
+ c_vaddr = (uint8_t *)m_vaddr + size;
+ c_dma = m_dma + size;
+ size += sizeof(cpt_res_s_t);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* Reserve memory for cpt request info */
+ req = m_vaddr;
+
+ size = sizeof(struct cpt_request_info);
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
+
+ /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
+ opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
+ (dir << 4) | (0 << 3) | (flags & 0x7));
+
+ /*
+ * GP op header, lengths are expected in bits.
+ */
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+ vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len);
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ /* consider iv len */
+ if (flags == 0x0) {
+ encr_offset += iv_len;
+ auth_offset += iv_len;
+ }
+
+ /* save space for offset ctrl and iv */
+ offset_vaddr = m_vaddr;
+ offset_dma = m_dma;
+
+ m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
+ m_dma += OFF_CTRL_LEN + iv_len;
+ m_size -= OFF_CTRL_LEN + iv_len;
+
+ /* DPTR has SG list */
+ in_buffer = m_vaddr;
+ dptr_dma = m_dma;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* TODO Add error check if space will be sufficient */
+ gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+ /*
+ * Input Gather List
+ */
+ i = 0;
+
+ /* Offset control word followed by iv */
+
+ if (flags == 0x0) {
+ inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
+ outputlen = inputlen;
+ /* iv offset is 0 */
+ *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+ } else {
+ inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
+ outputlen = mac_len;
+ /* iv offset is 0 */
+ *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset);
+ }
+
+ i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
+
+ /* IV */
+ iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN;
+ memcpy(iv_d, iv_s, iv_len);
+
+ /* input data */
+ size = inputlen - iv_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(gather_comp, i,
+ params->src_iov, 0,
+ &size, NULL, 0);
+
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+ g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ /*
+ * Output Scatter List
+ */
+
+ i = 0;
+ scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+ if (flags == 0x1) {
+ /* IV in SLIST only for F8 */
+ iv_len = 0;
+ }
+
+ /* IV */
+ if (iv_len) {
+ i = fill_sg_comp(scatter_comp, i,
+ offset_dma + OFF_CTRL_LEN,
+ iv_len);
+ }
+
+ /* Add output data */
+ if (req_flags & VALID_MAC_BUF) {
+ size = outputlen - iv_len - mac_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(scatter_comp, i,
+ params->dst_iov, 0,
+ &size, NULL, 0);
+
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+
+ /* mac data */
+ if (mac_len) {
+ i = fill_sg_comp_from_buf(scatter_comp, i,
+ &params->mac_buf);
+ }
+ } else {
+ /* Output including mac */
+ size = outputlen - iv_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(scatter_comp, i,
+ params->dst_iov, 0,
+ &size, NULL, 0);
+
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ }
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+ s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+ /* This is DPTR len incase of SG mode */
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* cpt alternate completion address saved earlier */
+ req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+ *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+ rptr_dma = c_dma - 8;
+
+ req->ist.ei1 = dptr_dma;
+ req->ist.ei2 = rptr_dma;
+
+ /* First 16-bit swap then 64-bit swap */
+ /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+ * to eliminate all the swapping
+ */
+ vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+ /* vq command w3 */
+ vq_cmd_w3.u64 = 0;
+ vq_cmd_w3.s.grp = 0;
+ vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
+ offsetof(struct cpt_ctx, k_ctx);
+
+ /* 16 byte aligned cpt res address */
+ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+ *req->completion_addr = COMPLETION_CODE_INIT;
+ req->comp_baddr = c_dma;
+
+ /* Fill microcode part of instruction */
+ req->ist.ei0 = vq_cmd_w0.u64;
+ req->ist.ei3 = vq_cmd_w3.u64;
+
+ req->op = op;
+
+ *prep_req = req;
+ return 0;
+}
+
+static __rte_always_inline int
+cpt_kasumi_dec_prep(uint64_t d_offs,
+ uint64_t d_lens,
+ fc_params_t *params,
+ void *op,
+ void **prep_req)
+{
+ uint32_t size;
+ int32_t inputlen = 0, outputlen;
+ struct cpt_ctx *cpt_ctx;
+ uint8_t i = 0, iv_len = 8;
+ struct cpt_request_info *req;
+ buf_ptr_t *buf_p;
+ uint32_t encr_offset;
+ uint32_t encr_data_len;
+ int flags, m_size;
+ uint8_t dir = 0;
+ void *m_vaddr, *c_vaddr;
+ uint64_t m_dma, c_dma;
+ uint64_t *offset_vaddr, offset_dma;
+ vq_cmd_word0_t vq_cmd_w0;
+ vq_cmd_word3_t vq_cmd_w3;
+ opcode_info_t opcode;
+ uint8_t *in_buffer;
+ uint32_t g_size_bytes, s_size_bytes;
+ uint64_t dptr_dma, rptr_dma;
+ sg_comp_t *gather_comp;
+ sg_comp_t *scatter_comp;
+
+ buf_p = &params->meta_buf;
+ m_vaddr = buf_p->vaddr;
+ m_dma = buf_p->dma_addr;
+ m_size = buf_p->size;
+
+ encr_offset = ENCR_OFFSET(d_offs) / 8;
+ encr_data_len = ENCR_DLEN(d_lens);
+
+ cpt_ctx = params->ctx_buf.vaddr;
+ flags = cpt_ctx->zsk_flags;
+ /*
+ * Save initial space that followed app data for completion code &
+ * alternate completion code to fall in same cache line as app data
+ */
+ m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+ m_dma += COMPLETION_CODE_SIZE;
+ size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+ (uint8_t *)m_vaddr;
+
+ c_vaddr = (uint8_t *)m_vaddr + size;
+ c_dma = m_dma + size;
+ size += sizeof(cpt_res_s_t);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* Reserve memory for cpt request info */
+ req = m_vaddr;
+
+ size = sizeof(struct cpt_request_info);
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
+
+ /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
+ opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
+ (dir << 4) | (0 << 3) | (flags & 0x7));
+
+ /*
+ * GP op header, lengths are expected in bits.
+ */
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ /* consider iv len */
+ encr_offset += iv_len;
+
+ inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8);
+ outputlen = inputlen;
+
+ /* save space for offset ctrl & iv */
+ offset_vaddr = m_vaddr;
+ offset_dma = m_dma;
+
+ m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
+ m_dma += OFF_CTRL_LEN + iv_len;
+ m_size -= OFF_CTRL_LEN + iv_len;
+
+ /* DPTR has SG list */
+ in_buffer = m_vaddr;
+ dptr_dma = m_dma;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* TODO Add error check if space will be sufficient */
+ gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+ /*
+ * Input Gather List
+ */
+ i = 0;
+
+ /* Offset control word followed by iv */
+ *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+
+ i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
+
+ /* IV */
+ memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN,
+ params->iv_buf, iv_len);
+
+ /* Add input data */
+ size = inputlen - iv_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(gather_comp, i,
+ params->src_iov,
+ 0, &size, NULL, 0);
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+ g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ /*
+ * Output Scatter List
+ */
+
+ i = 0;
+ scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+ /* IV */
+ i = fill_sg_comp(scatter_comp, i,
+ offset_dma + OFF_CTRL_LEN,
+ iv_len);
+
+ /* Add output data */
+ size = outputlen - iv_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(scatter_comp, i,
+ params->dst_iov, 0,
+ &size, NULL, 0);
+ if (size)
+ return ERR_BAD_INPUT_ARG;
+ }
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+ s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+ /* This is DPTR len incase of SG mode */
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* cpt alternate completion address saved earlier */
+ req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+ *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+ rptr_dma = c_dma - 8;
+
+ req->ist.ei1 = dptr_dma;
+ req->ist.ei2 = rptr_dma;
+
+ /* First 16-bit swap then 64-bit swap */
+ /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+ * to eliminate all the swapping
+ */
+ vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+ /* vq command w3 */
+ vq_cmd_w3.u64 = 0;
+ vq_cmd_w3.s.grp = 0;
+ vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
+ offsetof(struct cpt_ctx, k_ctx);
+
+ /* 16 byte aligned cpt res address */
+ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+ *req->completion_addr = COMPLETION_CODE_INIT;
+ req->comp_baddr = c_dma;
+
+ /* Fill microcode part of instruction */
+ req->ist.ei0 = vq_cmd_w0.u64;
+ req->ist.ei3 = vq_cmd_w3.u64;
+
+ req->op = op;
+
+ *prep_req = req;
+ return 0;
+}
+
+static __rte_always_inline void *
+cpt_fc_dec_hmac_prep(uint32_t flags,
+ uint64_t d_offs,
+ uint64_t d_lens,
+ fc_params_t *fc_params,
+ void *op, int *ret_val)
+{
+ struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
+ uint8_t fc_type;
+ void *prep_req = NULL;
+ int ret;
+
+ fc_type = ctx->fc_type;
+
+ if (likely(fc_type == FC_GEN)) {
+ ret = cpt_dec_hmac_prep(flags, d_offs, d_lens,
+ fc_params, op, &prep_req);
+ } else if (fc_type == ZUC_SNOW3G) {
+ ret = cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens,
+ fc_params, op, &prep_req);
+ } else if (fc_type == KASUMI) {
+ ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op,
+ &prep_req);
+ } else {
+ /*
+ * For AUTH_ONLY case,
+ * MC only supports digest generation and verification
+ * should be done in software by memcmp()
+ */
+
+ ret = ERR_EIO;
+ }
+
+ if (unlikely(!prep_req))
+ *ret_val = ret;
+ return prep_req;
+}
+
+static __rte_always_inline void *__hot
+cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
+ fc_params_t *fc_params, void *op, int *ret_val)
+{
+ struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
+ uint8_t fc_type;
+ void *prep_req = NULL;
+ int ret;
+
+ fc_type = ctx->fc_type;
+
+ /* Common api for rest of the ops */
+ if (likely(fc_type == FC_GEN)) {
+ ret = cpt_enc_hmac_prep(flags, d_offs, d_lens,
+ fc_params, op, &prep_req);
+ } else if (fc_type == ZUC_SNOW3G) {
+ ret = cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens,
+ fc_params, op, &prep_req);
+ } else if (fc_type == KASUMI) {
+ ret = cpt_kasumi_enc_prep(flags, d_offs, d_lens,
+ fc_params, op, &prep_req);
+ } else if (fc_type == HASH_HMAC) {
+ ret = cpt_digest_gen_prep(flags, d_lens, fc_params, op,
+ &prep_req);
+ } else {
+ ret = ERR_EIO;
+ }
+
+ if (unlikely(!prep_req))
+ *ret_val = ret;
+ return prep_req;
+}
+
+static __rte_always_inline int
+cpt_fc_auth_set_key(void *ctx, auth_type_t type, uint8_t *key,
+ uint16_t key_len, uint16_t mac_len)
+{
+ struct cpt_ctx *cpt_ctx = ctx;
+ mc_fc_context_t *fctx = &cpt_ctx->fctx;
+ uint64_t *ctrl_flags = NULL;
+
+ if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) {
+ uint32_t keyx[4];
+
+ if (key_len != 16)
+ return -1;
+ /* No support for AEAD yet */
+ if (cpt_ctx->enc_cipher)
+ return -1;
+ /* For ZUC/SNOW3G/Kasumi */
+ switch (type) {
+ case SNOW3G_UIA2:
+ cpt_ctx->snow3g = 1;
+ gen_key_snow3g(key, keyx);
+ memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len);
+ cpt_ctx->fc_type = ZUC_SNOW3G;
+ cpt_ctx->zsk_flags = 0x1;
+ break;
+ case ZUC_EIA3:
+ cpt_ctx->snow3g = 0;
+ memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len);
+ memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32);
+ cpt_ctx->fc_type = ZUC_SNOW3G;
+ cpt_ctx->zsk_flags = 0x1;
+ break;
+ case KASUMI_F9_ECB:
+ /* Kasumi ECB mode */
+ cpt_ctx->k_ecb = 1;
+ memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
+ cpt_ctx->fc_type = KASUMI;
+ cpt_ctx->zsk_flags = 0x1;
+ break;
+ case KASUMI_F9_CBC:
+ memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
+ cpt_ctx->fc_type = KASUMI;
+ cpt_ctx->zsk_flags = 0x1;
+ break;
+ default:
+ return -1;
+ }
+ cpt_ctx->mac_len = 4;
+ cpt_ctx->hash_type = type;
+ return 0;
+ }
+
+ if (!(cpt_ctx->fc_type == FC_GEN && !type)) {
+ if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher)
+ cpt_ctx->fc_type = HASH_HMAC;
+ }
+
+ ctrl_flags = (uint64_t *)&fctx->enc.enc_ctrl.flags;
+ *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags);
+
+ /* For GMAC auth, cipher must be NULL */
+ if (type == GMAC_TYPE)
+ CPT_P_ENC_CTRL(fctx).enc_cipher = 0;
+
+ CPT_P_ENC_CTRL(fctx).hash_type = cpt_ctx->hash_type = type;
+ CPT_P_ENC_CTRL(fctx).mac_len = cpt_ctx->mac_len = mac_len;
+
+ if (key_len) {
+ cpt_ctx->hmac = 1;
+ memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key));
+ memcpy(cpt_ctx->auth_key, key, key_len);
+ cpt_ctx->auth_key_len = key_len;
+ memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
+ memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad));
+ memcpy(fctx->hmac.opad, key, key_len);
+ CPT_P_ENC_CTRL(fctx).auth_input_type = 1;
+ }
+ *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags);
+ return 0;
+}
+
+static __rte_always_inline int
+fill_sess_aead(struct rte_crypto_sym_xform *xform,
+ struct cpt_sess_misc *sess)
+{
+ struct rte_crypto_aead_xform *aead_form;
+ cipher_type_t enc_type = 0; /* NULL Cipher type */
+ auth_type_t auth_type = 0; /* NULL Auth type */
+ uint32_t cipher_key_len = 0;
+ uint8_t zsk_flag = 0, aes_gcm = 0;
+ aead_form = &xform->aead;
+ void *ctx;
+
+ if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
+ aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) {
+ sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
+ sess->cpt_op |= CPT_OP_AUTH_GENERATE;
+ } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
+ aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) {
+ sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
+ sess->cpt_op |= CPT_OP_AUTH_VERIFY;
+ } else {
+ CPT_LOG_DP_ERR("Unknown cipher operation\n");
+ return -1;
+ }
+ switch (aead_form->algo) {
+ case RTE_CRYPTO_AEAD_AES_GCM:
+ enc_type = AES_GCM;
+ cipher_key_len = 16;
+ aes_gcm = 1;
+ break;
+ case RTE_CRYPTO_AEAD_AES_CCM:
+ CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
+ aead_form->algo);
+ return -1;
+ default:
+ CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
+ aead_form->algo);
+ return -1;
+ }
+ if (aead_form->key.length < cipher_key_len) {
+ CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
+ (unsigned int long)aead_form->key.length);
+ return -1;
+ }
+ sess->zsk_flag = zsk_flag;
+ sess->aes_gcm = aes_gcm;
+ sess->mac_len = aead_form->digest_length;
+ sess->iv_offset = aead_form->iv.offset;
+ sess->iv_length = aead_form->iv.length;
+ sess->aad_length = aead_form->aad_length;
+ ctx = (void *)((uint8_t *)sess + sizeof(struct cpt_sess_misc)),
+
+ cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data,
+ aead_form->key.length, NULL);
+
+ cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, aead_form->digest_length);
+
+ return 0;
+}
+
+static __rte_always_inline int
+fill_sess_cipher(struct rte_crypto_sym_xform *xform,
+ struct cpt_sess_misc *sess)
+{
+ struct rte_crypto_cipher_xform *c_form;
+ cipher_type_t enc_type = 0; /* NULL Cipher type */
+ uint32_t cipher_key_len = 0;
+ uint8_t zsk_flag = 0, aes_gcm = 0, aes_ctr = 0, is_null = 0;
+
+ if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER)
+ return -1;
+
+ c_form = &xform->cipher;
+
+ if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+ sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
+ else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT)
+ sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
+ else {
+ CPT_LOG_DP_ERR("Unknown cipher operation\n");
+ return -1;
+ }
+
+ switch (c_form->algo) {
+ case RTE_CRYPTO_CIPHER_AES_CBC:
+ enc_type = AES_CBC;
+ cipher_key_len = 16;
+ break;
+ case RTE_CRYPTO_CIPHER_3DES_CBC:
+ enc_type = DES3_CBC;
+ cipher_key_len = 24;
+ break;
+ case RTE_CRYPTO_CIPHER_DES_CBC:
+ /* DES is implemented using 3DES in hardware */
+ enc_type = DES3_CBC;
+ cipher_key_len = 8;
+ break;
+ case RTE_CRYPTO_CIPHER_AES_CTR:
+ enc_type = AES_CTR;
+ cipher_key_len = 16;
+ aes_ctr = 1;
+ break;
+ case RTE_CRYPTO_CIPHER_NULL:
+ enc_type = 0;
+ is_null = 1;
+ break;
+ case RTE_CRYPTO_CIPHER_KASUMI_F8:
+ enc_type = KASUMI_F8_ECB;
+ cipher_key_len = 16;
+ zsk_flag = K_F8;
+ break;
+ case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
+ enc_type = SNOW3G_UEA2;
+ cipher_key_len = 16;
+ zsk_flag = ZS_EA;
+ break;
+ case RTE_CRYPTO_CIPHER_ZUC_EEA3:
+ enc_type = ZUC_EEA3;
+ cipher_key_len = 16;
+ zsk_flag = ZS_EA;
+ break;
+ case RTE_CRYPTO_CIPHER_AES_XTS:
+ enc_type = AES_XTS;
+ cipher_key_len = 16;
+ break;
+ case RTE_CRYPTO_CIPHER_3DES_ECB:
+ enc_type = DES3_ECB;
+ cipher_key_len = 24;
+ break;
+ case RTE_CRYPTO_CIPHER_AES_ECB:
+ enc_type = AES_ECB;
+ cipher_key_len = 16;
+ break;
+ case RTE_CRYPTO_CIPHER_3DES_CTR:
+ case RTE_CRYPTO_CIPHER_AES_F8:
+ case RTE_CRYPTO_CIPHER_ARC4:
+ CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
+ c_form->algo);
+ return -1;
+ default:
+ CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
+ c_form->algo);
+ return -1;
+ }
+
+ if (c_form->key.length < cipher_key_len) {
+ CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
+ (unsigned long) c_form->key.length);
+ return -1;
+ }
+
+ sess->zsk_flag = zsk_flag;
+ sess->aes_gcm = aes_gcm;
+ sess->aes_ctr = aes_ctr;
+ sess->iv_offset = c_form->iv.offset;
+ sess->iv_length = c_form->iv.length;
+ sess->is_null = is_null;
+
+ cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, c_form->key.data,
+ c_form->key.length, NULL);
+
+ return 0;
+}
+
+static __rte_always_inline int
+fill_sess_auth(struct rte_crypto_sym_xform *xform,
+ struct cpt_sess_misc *sess)
+{
+ struct rte_crypto_auth_xform *a_form;
+ auth_type_t auth_type = 0; /* NULL Auth type */
+ uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0;
+
+ if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH)
+ goto error_out;
+
+ a_form = &xform->auth;
+
+ if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
+ sess->cpt_op |= CPT_OP_AUTH_VERIFY;
+ else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
+ sess->cpt_op |= CPT_OP_AUTH_GENERATE;
+ else {
+ CPT_LOG_DP_ERR("Unknown auth operation");
+ return -1;
+ }
+
+ if (a_form->key.length > 64) {
+ CPT_LOG_DP_ERR("Auth key length is big");
+ return -1;
+ }
+
+ switch (a_form->algo) {
+ case RTE_CRYPTO_AUTH_SHA1_HMAC:
+ /* Fall through */
+ case RTE_CRYPTO_AUTH_SHA1:
+ auth_type = SHA1_TYPE;
+ break;
+ case RTE_CRYPTO_AUTH_SHA256_HMAC:
+ case RTE_CRYPTO_AUTH_SHA256:
+ auth_type = SHA2_SHA256;
+ break;
+ case RTE_CRYPTO_AUTH_SHA512_HMAC:
+ case RTE_CRYPTO_AUTH_SHA512:
+ auth_type = SHA2_SHA512;
+ break;
+ case RTE_CRYPTO_AUTH_AES_GMAC:
+ auth_type = GMAC_TYPE;
+ aes_gcm = 1;
+ break;
+ case RTE_CRYPTO_AUTH_SHA224_HMAC:
+ case RTE_CRYPTO_AUTH_SHA224:
+ auth_type = SHA2_SHA224;
+ break;
+ case RTE_CRYPTO_AUTH_SHA384_HMAC:
+ case RTE_CRYPTO_AUTH_SHA384:
+ auth_type = SHA2_SHA384;
+ break;
+ case RTE_CRYPTO_AUTH_MD5_HMAC:
+ case RTE_CRYPTO_AUTH_MD5:
+ auth_type = MD5_TYPE;
+ break;
+ case RTE_CRYPTO_AUTH_KASUMI_F9:
+ auth_type = KASUMI_F9_ECB;
+ /*
+ * Indicate that direction needs to be taken out
+ * from end of src
+ */
+ zsk_flag = K_F9;
+ break;
+ case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
+ auth_type = SNOW3G_UIA2;
+ zsk_flag = ZS_IA;
+ break;
+ case RTE_CRYPTO_AUTH_ZUC_EIA3:
+ auth_type = ZUC_EIA3;
+ zsk_flag = ZS_IA;
+ break;
+ case RTE_CRYPTO_AUTH_NULL:
+ auth_type = 0;
+ is_null = 1;
+ break;
+ case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
+ case RTE_CRYPTO_AUTH_AES_CMAC:
+ case RTE_CRYPTO_AUTH_AES_CBC_MAC:
+ CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u",
+ a_form->algo);
+ goto error_out;
+ default:
+ CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified",
+ a_form->algo);
+ goto error_out;
+ }
+
+ sess->zsk_flag = zsk_flag;
+ sess->aes_gcm = aes_gcm;
+ sess->mac_len = a_form->digest_length;
+ sess->is_null = is_null;
+ if (zsk_flag) {
+ sess->auth_iv_offset = a_form->iv.offset;
+ sess->auth_iv_length = a_form->iv.length;
+ }
+ cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, a_form->key.data,
+ a_form->key.length, a_form->digest_length);
+
+ return 0;
+
+error_out:
+ return -1;
+}
+
+static __rte_always_inline int
+fill_sess_gmac(struct rte_crypto_sym_xform *xform,
+ struct cpt_sess_misc *sess)
+{
+ struct rte_crypto_auth_xform *a_form;
+ cipher_type_t enc_type = 0; /* NULL Cipher type */
+ auth_type_t auth_type = 0; /* NULL Auth type */
+ uint8_t zsk_flag = 0, aes_gcm = 0;
+ void *ctx;
+
+ if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH)
+ return -1;
+
+ a_form = &xform->auth;
+
+ if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
+ sess->cpt_op |= CPT_OP_ENCODE;
+ else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
+ sess->cpt_op |= CPT_OP_DECODE;
+ else {
+ CPT_LOG_DP_ERR("Unknown auth operation");
+ return -1;
+ }
+
+ switch (a_form->algo) {
+ case RTE_CRYPTO_AUTH_AES_GMAC:
+ enc_type = AES_GCM;
+ auth_type = GMAC_TYPE;
+ break;
+ default:
+ CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
+ a_form->algo);
+ return -1;
+ }
+
+ sess->zsk_flag = zsk_flag;
+ sess->aes_gcm = aes_gcm;
+ sess->is_gmac = 1;
+ sess->iv_offset = a_form->iv.offset;
+ sess->iv_length = a_form->iv.length;
+ sess->mac_len = a_form->digest_length;
+ ctx = (void *)((uint8_t *)sess + sizeof(struct cpt_sess_misc)),
+
+ cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data,
+ a_form->key.length, NULL);
+ cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, a_form->digest_length);
+
+ return 0;
+}
+
+static __rte_always_inline void *
+alloc_op_meta(struct rte_mbuf *m_src,
+ buf_ptr_t *buf,
+ int32_t len,
+ struct rte_mempool *cpt_meta_pool)
+{
+ uint8_t *mdata;
+
+#ifndef CPT_ALWAYS_USE_SEPARATE_BUF
+ if (likely(m_src && (m_src->nb_segs == 1))) {
+ int32_t tailroom;
+ phys_addr_t mphys;
+
+ /* Check if tailroom is sufficient to hold meta data */
+ tailroom = rte_pktmbuf_tailroom(m_src);
+ if (likely(tailroom > len + 8)) {
+ mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len;
+ mphys = m_src->buf_physaddr + m_src->buf_len;
+ mdata -= len;
+ mphys -= len;
+ buf->vaddr = mdata;
+ buf->dma_addr = mphys;
+ buf->size = len;
+ /* Indicate that this is a mbuf allocated mdata */
+ mdata = (uint8_t *)((uint64_t)mdata | 1ull);
+ return mdata;
+ }
+ }
+#else
+ RTE_SET_USED(m_src);
+#endif
+
+ if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
+ return NULL;
+
+ buf->vaddr = mdata;
+ buf->dma_addr = rte_mempool_virt2iova(mdata);
+ buf->size = len;
+
+ return mdata;
+}
+
+/**
+ * cpt_free_metabuf - free metabuf to mempool.
+ * @param instance: pointer to instance.
+ * @param objp: pointer to the metabuf.
+ */
+static __rte_always_inline void
+free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool)
+{
+ bool nofree = ((uintptr_t)mdata & 1ull);
+
+ if (likely(nofree))
+ return;
+ rte_mempool_put(cpt_meta_pool, mdata);
+}
+
+static __rte_always_inline uint32_t
+prepare_iov_from_pkt(struct rte_mbuf *pkt,
+ iov_ptr_t *iovec, uint32_t start_offset)
+{
+ uint16_t index = 0;
+ void *seg_data = NULL;
+ phys_addr_t seg_phys;
+ int32_t seg_size = 0;
+
+ if (!pkt) {
+ iovec->buf_cnt = 0;
+ return 0;
+ }
+
+ if (!start_offset) {
+ seg_data = rte_pktmbuf_mtod(pkt, void *);
+ seg_phys = rte_pktmbuf_mtophys(pkt);
+ seg_size = pkt->data_len;
+ } else {
+ while (start_offset >= pkt->data_len) {
+ start_offset -= pkt->data_len;
+ pkt = pkt->next;
+ }
+
+ seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset);
+ seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset);
+ seg_size = pkt->data_len - start_offset;
+ if (!seg_size)
+ return 1;
+ }
+
+ /* first seg */
+ iovec->bufs[index].vaddr = seg_data;
+ iovec->bufs[index].dma_addr = seg_phys;
+ iovec->bufs[index].size = seg_size;
+ index++;
+ pkt = pkt->next;
+
+ while (unlikely(pkt != NULL)) {
+ seg_data = rte_pktmbuf_mtod(pkt, void *);
+ seg_phys = rte_pktmbuf_mtophys(pkt);
+ seg_size = pkt->data_len;
+ if (!seg_size)
+ break;
+
+ iovec->bufs[index].vaddr = seg_data;
+ iovec->bufs[index].dma_addr = seg_phys;
+ iovec->bufs[index].size = seg_size;
+
+ index++;
+
+ pkt = pkt->next;
+ }
+
+ iovec->buf_cnt = index;
+ return 0;
+}
+
+static __rte_always_inline uint32_t
+prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt,
+ fc_params_t *param,
+ uint32_t *flags)
+{
+ uint16_t index = 0;
+ void *seg_data = NULL;
+ phys_addr_t seg_phys;
+ uint32_t seg_size = 0;
+ iov_ptr_t *iovec;
+
+ seg_data = rte_pktmbuf_mtod(pkt, void *);
+ seg_phys = rte_pktmbuf_mtophys(pkt);
+ seg_size = pkt->data_len;
+
+ /* first seg */
+ if (likely(!pkt->next)) {
+ uint32_t headroom, tailroom;
+
+ *flags |= SINGLE_BUF_INPLACE;
+ headroom = rte_pktmbuf_headroom(pkt);
+ tailroom = rte_pktmbuf_tailroom(pkt);
+ if (likely((headroom >= 24) &&
+ (tailroom >= 8))) {
+ /* In 83XX this is prerequivisit for Direct mode */
+ *flags |= SINGLE_BUF_HEADTAILROOM;
+ }
+ param->bufs[0].vaddr = seg_data;
+ param->bufs[0].dma_addr = seg_phys;
+ param->bufs[0].size = seg_size;
+ return 0;
+ }
+ iovec = param->src_iov;
+ iovec->bufs[index].vaddr = seg_data;
+ iovec->bufs[index].dma_addr = seg_phys;
+ iovec->bufs[index].size = seg_size;
+ index++;
+ pkt = pkt->next;
+
+ while (unlikely(pkt != NULL)) {
+ seg_data = rte_pktmbuf_mtod(pkt, void *);
+ seg_phys = rte_pktmbuf_mtophys(pkt);
+ seg_size = pkt->data_len;
+
+ if (!seg_size)
+ break;
+
+ iovec->bufs[index].vaddr = seg_data;
+ iovec->bufs[index].dma_addr = seg_phys;
+ iovec->bufs[index].size = seg_size;
+
+ index++;
+
+ pkt = pkt->next;
+ }
+
+ iovec->buf_cnt = index;
+ return 0;
+}
+
+static __rte_always_inline void *
+fill_fc_params(struct rte_crypto_op *cop,
+ struct cpt_sess_misc *sess_misc,
+ void **mdata_ptr,
+ int *op_ret)
+{
+ uint32_t space = 0;
+ struct rte_crypto_sym_op *sym_op = cop->sym;
+ void *mdata;
+ uintptr_t *op;
+ uint32_t mc_hash_off;
+ uint32_t flags = 0;
+ uint64_t d_offs, d_lens;
+ void *prep_req = NULL;
+ struct rte_mbuf *m_src, *m_dst;
+ uint8_t cpt_op = sess_misc->cpt_op;
+ uint8_t zsk_flag = sess_misc->zsk_flag;
+ uint8_t aes_gcm = sess_misc->aes_gcm;
+ uint16_t mac_len = sess_misc->mac_len;
+#ifdef CPT_ALWAYS_USE_SG_MODE
+ uint8_t inplace = 0;
+#else
+ uint8_t inplace = 1;
+#endif
+ fc_params_t fc_params;
+ char src[SRC_IOV_SIZE];
+ char dst[SRC_IOV_SIZE];
+ uint32_t iv_buf[4];
+ struct cptvf_meta_info *cpt_m_info =
+ (struct cptvf_meta_info *)(*mdata_ptr);
+
+ if (likely(sess_misc->iv_length)) {
+ flags |= VALID_IV_BUF;
+ fc_params.iv_buf = rte_crypto_op_ctod_offset(cop,
+ uint8_t *, sess_misc->iv_offset);
+ if (sess_misc->aes_ctr &&
+ unlikely(sess_misc->iv_length != 16)) {
+ memcpy((uint8_t *)iv_buf,
+ rte_crypto_op_ctod_offset(cop,
+ uint8_t *, sess_misc->iv_offset), 12);
+ iv_buf[3] = rte_cpu_to_be_32(0x1);
+ fc_params.iv_buf = iv_buf;
+ }
+ }
+
+ if (zsk_flag) {
+ fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
+ uint8_t *,
+ sess_misc->auth_iv_offset);
+ if (zsk_flag == K_F9) {
+ CPT_LOG_DP_ERR("Should not reach here for "
+ "kasumi F9\n");
+ }
+ if (zsk_flag != ZS_EA)
+ inplace = 0;
+ }
+ m_src = sym_op->m_src;
+ m_dst = sym_op->m_dst;
+
+ if (aes_gcm) {
+ uint8_t *salt;
+ uint8_t *aad_data;
+ uint16_t aad_len;
+
+ d_offs = sym_op->aead.data.offset;
+ d_lens = sym_op->aead.data.length;
+ mc_hash_off = sym_op->aead.data.offset +
+ sym_op->aead.data.length;
+
+ aad_data = sym_op->aead.aad.data;
+ aad_len = sess_misc->aad_length;
+ if (likely((aad_data + aad_len) ==
+ rte_pktmbuf_mtod_offset(m_src,
+ uint8_t *,
+ sym_op->aead.data.offset))) {
+ d_offs = (d_offs - aad_len) | (d_offs << 16);
+ d_lens = (d_lens + aad_len) | (d_lens << 32);
+ } else {
+ fc_params.aad_buf.vaddr = sym_op->aead.aad.data;
+ fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr;
+ fc_params.aad_buf.size = aad_len;
+ flags |= VALID_AAD_BUF;
+ inplace = 0;
+ d_offs = d_offs << 16;
+ d_lens = d_lens << 32;
+ }
+
+ salt = fc_params.iv_buf;
+ if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
+ cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
+ sess_misc->salt = *(uint32_t *)salt;
+ }
+ fc_params.iv_buf = salt + 4;
+ if (likely(mac_len)) {
+ struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst :
+ m_src;
+
+ if (!m)
+ m = m_src;
+
+ /* hmac immediately following data is best case */
+ if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
+ mc_hash_off !=
+ (uint8_t *)sym_op->aead.digest.data)) {
+ flags |= VALID_MAC_BUF;
+ fc_params.mac_buf.size = sess_misc->mac_len;
+ fc_params.mac_buf.vaddr =
+ sym_op->aead.digest.data;
+ fc_params.mac_buf.dma_addr =
+ sym_op->aead.digest.phys_addr;
+ inplace = 0;
+ }
+ }
+ } else {
+ d_offs = sym_op->cipher.data.offset;
+ d_lens = sym_op->cipher.data.length;
+ mc_hash_off = sym_op->cipher.data.offset +
+ sym_op->cipher.data.length;
+ d_offs = (d_offs << 16) | sym_op->auth.data.offset;
+ d_lens = (d_lens << 32) | sym_op->auth.data.length;
+
+ if (mc_hash_off < (sym_op->auth.data.offset +
+ sym_op->auth.data.length)){
+ mc_hash_off = (sym_op->auth.data.offset +
+ sym_op->auth.data.length);
+ }
+ /* for gmac, salt should be updated like in gcm */
+ if (unlikely(sess_misc->is_gmac)) {
+ uint8_t *salt;
+ salt = fc_params.iv_buf;
+ if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
+ cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
+ sess_misc->salt = *(uint32_t *)salt;
+ }
+ fc_params.iv_buf = salt + 4;
+ }
+ if (likely(mac_len)) {
+ struct rte_mbuf *m;
+
+ m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src;
+ if (!m)
+ m = m_src;
+
+ /* hmac immediately following data is best case */
+ if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
+ mc_hash_off !=
+ (uint8_t *)sym_op->auth.digest.data)) {
+ flags |= VALID_MAC_BUF;
+ fc_params.mac_buf.size =
+ sess_misc->mac_len;
+ fc_params.mac_buf.vaddr =
+ sym_op->auth.digest.data;
+ fc_params.mac_buf.dma_addr =
+ sym_op->auth.digest.phys_addr;
+ inplace = 0;
+ }
+ }
+ }
+ fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc);
+ fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr;
+
+ if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE))
+ inplace = 0;
+
+ if (likely(!m_dst && inplace)) {
+ /* Case of single buffer without AAD buf or
+ * separate mac buf in place and
+ * not air crypto
+ */
+ fc_params.dst_iov = fc_params.src_iov = (void *)src;
+
+ if (unlikely(prepare_iov_from_pkt_inplace(m_src,
+ &fc_params,
+ &flags))) {
+ CPT_LOG_DP_ERR("Prepare inplace src iov failed");
+ *op_ret = -1;
+ return NULL;
+ }
+
+ } else {
+ /* Out of place processing */
+ fc_params.src_iov = (void *)src;
+ fc_params.dst_iov = (void *)dst;
+
+ /* Store SG I/O in the api for reuse */
+ if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) {
+ CPT_LOG_DP_ERR("Prepare src iov failed");
+ *op_ret = -1;
+ return NULL;
+ }
+
+ if (unlikely(m_dst != NULL)) {
+ uint32_t pkt_len;
+
+ /* Try to make room as much as src has */
+ m_dst = sym_op->m_dst;
+ pkt_len = rte_pktmbuf_pkt_len(m_dst);
+
+ if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) {
+ pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len;
+ if (!rte_pktmbuf_append(m_dst, pkt_len)) {
+ CPT_LOG_DP_ERR("Not enough space in "
+ "m_dst %p, need %u"
+ " more",
+ m_dst, pkt_len);
+ return NULL;
+ }
+ }
+
+ if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) {
+ CPT_LOG_DP_ERR("Prepare dst iov failed for "
+ "m_dst %p", m_dst);
+ return NULL;
+ }
+ } else {
+ fc_params.dst_iov = (void *)src;
+ }
+ }
+
+ if (likely(flags & SINGLE_BUF_HEADTAILROOM))
+ mdata = alloc_op_meta(m_src,
+ &fc_params.meta_buf,
+ cpt_m_info->cptvf_op_sb_mlen,
+ cpt_m_info->cptvf_meta_pool);
+ else
+ mdata = alloc_op_meta(NULL,
+ &fc_params.meta_buf,
+ cpt_m_info->cptvf_op_mlen,
+ cpt_m_info->cptvf_meta_pool);
+
+ if (unlikely(mdata == NULL)) {
+ CPT_LOG_DP_ERR("Error allocating meta buffer for request");
+ return NULL;
+ }
+
+ op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull);
+ op[0] = (uintptr_t)mdata;
+ op[1] = (uintptr_t)cop;
+ op[2] = op[3] = 0; /* Used to indicate auth verify */
+ space += 4 * sizeof(uint64_t);
+
+ fc_params.meta_buf.vaddr = (uint8_t *)op + space;
+ fc_params.meta_buf.dma_addr += space;
+ fc_params.meta_buf.size -= space;
+
+ /* Finally prepare the instruction */
+ if (cpt_op & CPT_OP_ENCODE)
+ prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens,
+ &fc_params, op, op_ret);
+ else
+ prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens,
+ &fc_params, op, op_ret);
+
+ if (unlikely(!prep_req))
+ free_op_meta(mdata, cpt_m_info->cptvf_meta_pool);
+ *mdata_ptr = mdata;
+ return prep_req;
+}
+
+static __rte_always_inline void
+compl_auth_verify(struct rte_crypto_op *op,
+ uint8_t *gen_mac,
+ uint64_t mac_len)
+{
+ uint8_t *mac;
+ struct rte_crypto_sym_op *sym_op = op->sym;
+
+ if (sym_op->auth.digest.data)
+ mac = sym_op->auth.digest.data;
+ else
+ mac = rte_pktmbuf_mtod_offset(sym_op->m_src,
+ uint8_t *,
+ sym_op->auth.data.length +
+ sym_op->auth.data.offset);
+ if (!mac) {
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ return;
+ }
+
+ if (memcmp(mac, gen_mac, mac_len))
+ op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+ else
+ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+}
+
+static __rte_always_inline int
+instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess)
+{
+ struct rte_crypto_sym_xform *chain;
+
+ CPT_PMD_INIT_FUNC_TRACE();
+
+ if (cpt_is_algo_supported(xform))
+ goto err;
+
+ chain = xform;
+ while (chain) {
+ switch (chain->type) {
+ case RTE_CRYPTO_SYM_XFORM_AEAD:
+ if (fill_sess_aead(chain, sess))
+ goto err;
+ break;
+ case RTE_CRYPTO_SYM_XFORM_CIPHER:
+ if (fill_sess_cipher(chain, sess))
+ goto err;
+ break;
+ case RTE_CRYPTO_SYM_XFORM_AUTH:
+ if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
+ if (fill_sess_gmac(chain, sess))
+ goto err;
+ } else {
+ if (fill_sess_auth(chain, sess))
+ goto err;
+ }
+ break;
+ default:
+ CPT_LOG_DP_ERR("Invalid crypto xform type");
+ break;
+ }
+ chain = chain->next;
+ }
+
+ return 0;
+
+err:
+ return -1;
+}
+
+static __rte_always_inline void
+find_kasumif9_direction_and_length(uint8_t *src,
+ uint32_t counter_num_bytes,
+ uint32_t *addr_length_in_bits,
+ uint8_t *addr_direction)
+{
+ uint8_t found = 0;
+ while (!found && counter_num_bytes > 0) {
+ counter_num_bytes--;
+ if (src[counter_num_bytes] == 0x00)
+ continue;
+ if (src[counter_num_bytes] == 0x80) {
+ *addr_direction = src[counter_num_bytes - 1] & 0x1;
+ *addr_length_in_bits = counter_num_bytes * 8 - 1;
+ found = 1;
+ } else {
+ int i = 0;
+ uint8_t last_byte = src[counter_num_bytes];
+ for (i = 0; i < 8 && found == 0; i++) {
+ if (last_byte & (1 << i)) {
+ *addr_direction = (last_byte >> (i+1))
+ & 0x1;
+ if (i != 6)
+ *addr_length_in_bits =
+ counter_num_bytes * 8
+ + (8 - (i + 2));
+ else
+ *addr_length_in_bits =
+ counter_num_bytes * 8;
+ found = 1;
+ }
+ }
+ }
+ }
+}
+
+/*
+ * This handles all auth only except AES_GMAC
+ */
+static __rte_always_inline void *
+fill_digest_params(struct rte_crypto_op *cop,
+ struct cpt_sess_misc *sess,
+ void **mdata_ptr,
+ int *op_ret)
+{
+ uint32_t space = 0;
+ struct rte_crypto_sym_op *sym_op = cop->sym;
+ void *mdata;
+ phys_addr_t mphys;
+ uint64_t *op;
+ uint32_t auth_range_off;
+ uint32_t flags = 0;
+ uint64_t d_offs = 0, d_lens;
+ void *prep_req = NULL;
+ struct rte_mbuf *m_src, *m_dst;
+ uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK;
+ uint8_t zsk_flag = sess->zsk_flag;
+ uint16_t mac_len = sess->mac_len;
+ fc_params_t params;
+ char src[SRC_IOV_SIZE];
+ uint8_t iv_buf[16];
+ memset(&params, 0, sizeof(fc_params_t));
+ struct cptvf_meta_info *cpt_m_info =
+ (struct cptvf_meta_info *)(*mdata_ptr);
+
+ m_src = sym_op->m_src;
+
+ /* For just digest lets force mempool alloc */
+ mdata = alloc_op_meta(NULL, &params.meta_buf, cpt_m_info->cptvf_op_mlen,
+ cpt_m_info->cptvf_meta_pool);
+ if (mdata == NULL) {
+ CPT_LOG_DP_ERR("Error allocating meta buffer for request");
+ *op_ret = -ENOMEM;
+ return NULL;
+ }
+
+ mphys = params.meta_buf.dma_addr;
+
+ op = mdata;
+ op[0] = (uintptr_t)mdata;
+ op[1] = (uintptr_t)cop;
+ op[2] = op[3] = 0; /* Used to indicate auth verify */
+ space += 4 * sizeof(uint64_t);
+
+ auth_range_off = sym_op->auth.data.offset;
+
+ flags = VALID_MAC_BUF;
+ params.src_iov = (void *)src;
+ if (unlikely(zsk_flag)) {
+ /*
+ * Since for Zuc, Kasumi, Snow3g offsets are in bits
+ * we will send pass through even for auth only case,
+ * let MC handle it
+ */
+ d_offs = auth_range_off;
+ auth_range_off = 0;
+ params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
+ uint8_t *, sess->auth_iv_offset);
+ if (zsk_flag == K_F9) {
+ uint32_t length_in_bits, num_bytes;
+ uint8_t *src, direction = 0;
+ uint32_t counter_num_bytes;
+
+ memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src,
+ uint8_t *), 8);
+ /*
+ * This is kasumi f9, take direction from
+ * source buffer
+ */
+ length_in_bits = cop->sym->auth.data.length;
+ num_bytes = (length_in_bits >> 3);
+ counter_num_bytes = num_bytes;
+ src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *);
+ find_kasumif9_direction_and_length(src,
+ counter_num_bytes,
+ &length_in_bits,
+ &direction);
+ length_in_bits -= 64;
+ cop->sym->auth.data.offset += 64;
+ d_offs = cop->sym->auth.data.offset;
+ auth_range_off = d_offs / 8;
+ cop->sym->auth.data.length = length_in_bits;
+
+ /* Store it at end of auth iv */
+ iv_buf[8] = direction;
+ params.auth_iv_buf = iv_buf;
+ }
+ }
+
+ d_lens = sym_op->auth.data.length;
+
+ params.ctx_buf.vaddr = SESS_PRIV(sess);
+ params.ctx_buf.dma_addr = sess->ctx_dma_addr;
+
+ if (auth_op == CPT_OP_AUTH_GENERATE) {
+ if (sym_op->auth.digest.data) {
+ /*
+ * Digest to be generated
+ * in separate buffer
+ */
+ params.mac_buf.size =
+ sess->mac_len;
+ params.mac_buf.vaddr =
+ sym_op->auth.digest.data;
+ params.mac_buf.dma_addr =
+ sym_op->auth.digest.phys_addr;
+ } else {
+ uint32_t off = sym_op->auth.data.offset +
+ sym_op->auth.data.length;
+ int32_t dlen, space;
+
+ m_dst = sym_op->m_dst ?
+ sym_op->m_dst : sym_op->m_src;
+ dlen = rte_pktmbuf_pkt_len(m_dst);
+
+ space = off + mac_len - dlen;
+ if (space > 0)
+ if (!rte_pktmbuf_append(m_dst, space)) {
+ CPT_LOG_DP_ERR("Failed to extend "
+ "mbuf by %uB", space);
+ goto err;
+ }
+
+ params.mac_buf.vaddr =
+ rte_pktmbuf_mtod_offset(m_dst, void *, off);
+ params.mac_buf.dma_addr =
+ rte_pktmbuf_mtophys_offset(m_dst, off);
+ params.mac_buf.size = mac_len;
+ }
+ } else {
+ /* Need space for storing generated mac */
+ params.mac_buf.vaddr = (uint8_t *)mdata + space;
+ params.mac_buf.dma_addr = mphys + space;
+ params.mac_buf.size = mac_len;
+ space += RTE_ALIGN_CEIL(mac_len, 8);
+ op[2] = (uintptr_t)params.mac_buf.vaddr;
+ op[3] = mac_len;
+ }
+
+ params.meta_buf.vaddr = (uint8_t *)mdata + space;
+ params.meta_buf.dma_addr = mphys + space;
+ params.meta_buf.size -= space;
+
+ /* Out of place processing */
+ params.src_iov = (void *)src;
+
+ /*Store SG I/O in the api for reuse */
+ if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) {
+ CPT_LOG_DP_ERR("Prepare src iov failed");
+ *op_ret = -1;
+ goto err;
+ }
+
+ prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens,
+ &params, op, op_ret);
+ *mdata_ptr = mdata;
+ return prep_req;
+err:
+ if (unlikely(!prep_req))
+ free_op_meta(mdata, cpt_m_info->cptvf_meta_pool);
+ return NULL;
+}
+
+#endif /*_CPT_UCODE_H_ */
diff --git a/drivers/common/cpt/meson.build b/drivers/common/cpt/meson.build
new file mode 100644
index 00000000..0a905aa4
--- /dev/null
+++ b/drivers/common/cpt/meson.build
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium, Inc
+
+sources = files('cpt_pmd_ops_helper.c')
+
+deps = ['kvargs', 'pci', 'cryptodev']
+includes += include_directories('../../crypto/octeontx')
+allow_experimental_apis = true
diff --git a/drivers/common/cpt/rte_common_cpt_version.map b/drivers/common/cpt/rte_common_cpt_version.map
new file mode 100644
index 00000000..dec614f0
--- /dev/null
+++ b/drivers/common/cpt/rte_common_cpt_version.map
@@ -0,0 +1,6 @@
+DPDK_18.11 {
+ global:
+
+ cpt_pmd_ops_helper_get_mlen_direct_mode;
+ cpt_pmd_ops_helper_get_mlen_sg_mode;
+};
diff --git a/drivers/common/dpaax/Makefile b/drivers/common/dpaax/Makefile
new file mode 100644
index 00000000..94d2cf0c
--- /dev/null
+++ b/drivers/common/dpaax/Makefile
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 NXP
+#
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_common_dpaax.a
+
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+
+# versioning export map
+EXPORT_MAP := rte_common_dpaax_version.map
+
+# library version
+LIBABIVER := 1
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-y += dpaax_iova_table.c
+
+LDLIBS += -lrte_eal
+
+SYMLINK-y-include += dpaax_iova_table.h
+
+include $(RTE_SDK)/mk/rte.lib.mk \ No newline at end of file
diff --git a/drivers/common/dpaax/dpaax_iova_table.c b/drivers/common/dpaax/dpaax_iova_table.c
new file mode 100644
index 00000000..2dd38a92
--- /dev/null
+++ b/drivers/common/dpaax/dpaax_iova_table.c
@@ -0,0 +1,465 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#include <rte_memory.h>
+
+#include "dpaax_iova_table.h"
+#include "dpaax_logs.h"
+
+/* Global dpaax logger identifier */
+int dpaax_logger;
+
+/* Global table reference */
+struct dpaax_iova_table *dpaax_iova_table_p;
+
+static int dpaax_handle_memevents(void);
+
+/* A structure representing the device-tree node available in /proc/device-tree.
+ */
+struct reg_node {
+ phys_addr_t addr;
+ size_t len;
+};
+
+/* A ntohll equivalent routine
+ * XXX: This is only applicable for 64 bit environment.
+ */
+static void
+rotate_8(unsigned char *arr)
+{
+ uint32_t temp;
+ uint32_t *first_half;
+ uint32_t *second_half;
+
+ first_half = (uint32_t *)(arr);
+ second_half = (uint32_t *)(arr + 4);
+
+ temp = *first_half;
+ *first_half = *second_half;
+ *second_half = temp;
+
+ *first_half = ntohl(*first_half);
+ *second_half = ntohl(*second_half);
+}
+
+/* read_memory_nodes
+ * Memory layout for DPAAx platforms (LS1043, LS1046, LS1088, LS2088, LX2160)
+ * are populated by Uboot and available in device tree:
+ * /proc/device-tree/memory@<address>/reg <= register.
+ * Entries are of the form:
+ * (<8 byte start addr><8 byte length>)(..more similar blocks of start,len>)..
+ *
+ * @param count
+ * OUT populate number of entries found in memory node
+ * @return
+ * Pointer to array of reg_node elements, count size
+ */
+static struct reg_node *
+read_memory_node(unsigned int *count)
+{
+ int fd, ret, i;
+ unsigned int j;
+ glob_t result = {0};
+ struct stat statbuf = {0};
+ char file_data[MEM_NODE_FILE_LEN];
+ struct reg_node *nodes = NULL;
+
+ *count = 0;
+
+ ret = glob(MEM_NODE_PATH_GLOB, 0, NULL, &result);
+ if (ret != 0) {
+ DPAAX_DEBUG("Unable to glob device-tree memory node: (%s)(%d)",
+ MEM_NODE_PATH_GLOB, ret);
+ goto out;
+ }
+
+ if (result.gl_pathc != 1) {
+ /* Either more than one memory@<addr> node found, or none.
+ * In either case, cannot work ahead.
+ */
+ DPAAX_DEBUG("Found (%zu) entries in device-tree. Not supported!",
+ result.gl_pathc);
+ goto out;
+ }
+
+ DPAAX_DEBUG("Opening and parsing device-tree node: (%s)",
+ result.gl_pathv[0]);
+ fd = open(result.gl_pathv[0], O_RDONLY);
+ if (fd < 0) {
+ DPAAX_DEBUG("Unable to open the device-tree node: (%s)(fd=%d)",
+ MEM_NODE_PATH_GLOB, fd);
+ goto cleanup;
+ }
+
+ /* Stat to get the file size */
+ ret = fstat(fd, &statbuf);
+ if (ret != 0) {
+ DPAAX_DEBUG("Unable to get device-tree memory node size.");
+ goto cleanup;
+ }
+
+ DPAAX_DEBUG("Size of device-tree mem node: %lu", statbuf.st_size);
+ if (statbuf.st_size > MEM_NODE_FILE_LEN) {
+ DPAAX_DEBUG("More memory nodes available than assumed.");
+ DPAAX_DEBUG("System may not work properly!");
+ }
+
+ ret = read(fd, file_data, statbuf.st_size > MEM_NODE_FILE_LEN ?
+ MEM_NODE_FILE_LEN : statbuf.st_size);
+ if (ret <= 0) {
+ DPAAX_DEBUG("Unable to read device-tree memory node: (%d)",
+ ret);
+ goto cleanup;
+ }
+
+ /* The reg node should be multiple of 16 bytes, 8 bytes each for addr
+ * and len.
+ */
+ *count = (statbuf.st_size / 16);
+ if ((*count) <= 0 || (statbuf.st_size % 16 != 0)) {
+ DPAAX_DEBUG("Invalid memory node values or count. (size=%lu)",
+ statbuf.st_size);
+ goto cleanup;
+ }
+
+ /* each entry is of 16 bytes, and size/16 is total count of entries */
+ nodes = malloc(sizeof(struct reg_node) * (*count));
+ if (!nodes) {
+ DPAAX_DEBUG("Failure in allocating working memory.");
+ goto cleanup;
+ }
+ memset(nodes, 0, sizeof(struct reg_node) * (*count));
+
+ for (i = 0, j = 0; i < (statbuf.st_size) && j < (*count); i += 16, j++) {
+ memcpy(&nodes[j], file_data + i, 16);
+ /* Rotate (ntohl) each 8 byte entry */
+ rotate_8((unsigned char *)(&(nodes[j].addr)));
+ rotate_8((unsigned char *)(&(nodes[j].len)));
+ }
+
+ DPAAX_DEBUG("Device-tree memory node data:");
+ do {
+ DPAAX_DEBUG("\n %08" PRIx64 " %08zu", nodes[j].addr, nodes[j].len);
+ } while (--j);
+
+cleanup:
+ close(fd);
+ globfree(&result);
+out:
+ return nodes;
+}
+
+int
+dpaax_iova_table_populate(void)
+{
+ int ret;
+ unsigned int i, node_count;
+ size_t tot_memory_size, total_table_size;
+ struct reg_node *nodes;
+ struct dpaax_iovat_element *entry;
+
+ /* dpaax_iova_table_p is a singleton - only one instance should be
+ * created.
+ */
+ if (dpaax_iova_table_p) {
+ DPAAX_DEBUG("Multiple allocation attempt for IOVA Table (%p)",
+ dpaax_iova_table_p);
+ /* This can be an error case as well - some path not cleaning
+ * up table - but, for now, it is assumed that if IOVA Table
+ * pointer is valid, table is allocated.
+ */
+ return 0;
+ }
+
+ nodes = read_memory_node(&node_count);
+ if (nodes == NULL) {
+ DPAAX_WARN("PA->VA translation not available;");
+ DPAAX_WARN("Expect performance impact.");
+ return -1;
+ }
+
+ tot_memory_size = 0;
+ for (i = 0; i < node_count; i++)
+ tot_memory_size += nodes[i].len;
+
+ DPAAX_DEBUG("Total available PA memory size: %zu", tot_memory_size);
+
+ /* Total table size = meta data + tot_memory_size/8 */
+ total_table_size = sizeof(struct dpaax_iova_table) +
+ (sizeof(struct dpaax_iovat_element) * node_count) +
+ ((tot_memory_size / DPAAX_MEM_SPLIT) * sizeof(uint64_t));
+
+ /* TODO: This memory doesn't need to shared but needs to be always
+ * pinned to RAM (no swap out) - using hugepage rather than malloc
+ */
+ dpaax_iova_table_p = rte_zmalloc(NULL, total_table_size, 0);
+ if (dpaax_iova_table_p == NULL) {
+ DPAAX_WARN("Unable to allocate memory for PA->VA Table;");
+ DPAAX_WARN("PA->VA translation not available;");
+ DPAAX_WARN("Expect performance impact.");
+ free(nodes);
+ return -1;
+ }
+
+ /* Initialize table */
+ dpaax_iova_table_p->count = node_count;
+ entry = dpaax_iova_table_p->entries;
+
+ DPAAX_DEBUG("IOVA Table entries: (entry start = %p)", (void *)entry);
+ DPAAX_DEBUG("\t(entry),(start),(len),(next)");
+
+ for (i = 0; i < node_count; i++) {
+ /* dpaax_iova_table_p
+ * | dpaax_iova_table_p->entries
+ * | |
+ * | |
+ * V V
+ * +------+------+-------+---+----------+---------+---
+ * |iova_ |entry | entry | | pages | pages |
+ * |table | 1 | 2 |...| entry 1 | entry2 |
+ * +-----'+.-----+-------+---+;---------+;--------+---
+ * \ \ / /
+ * `~~~~~~|~~~~~>pages /
+ * \ /
+ * `~~~~~~~~~~~>pages
+ */
+ entry[i].start = nodes[i].addr;
+ entry[i].len = nodes[i].len;
+ if (i > 0)
+ entry[i].pages = entry[i-1].pages +
+ ((entry[i-1].len/DPAAX_MEM_SPLIT));
+ else
+ entry[i].pages = (uint64_t *)((unsigned char *)entry +
+ (sizeof(struct dpaax_iovat_element) *
+ node_count));
+
+ DPAAX_DEBUG("\t(%u),(%8"PRIx64"),(%8zu),(%8p)",
+ i, entry[i].start, entry[i].len, entry[i].pages);
+ }
+
+ /* Release memory associated with nodes array - not required now */
+ free(nodes);
+
+ DPAAX_DEBUG("Adding mem-event handler\n");
+ ret = dpaax_handle_memevents();
+ if (ret) {
+ DPAAX_ERR("Unable to add mem-event handler");
+ DPAAX_WARN("Cases with non-buffer pool mem won't work!");
+ }
+
+ return 0;
+}
+
+void
+dpaax_iova_table_depopulate(void)
+{
+ if (dpaax_iova_table_p == NULL)
+ return;
+
+ rte_free(dpaax_iova_table_p->entries);
+ dpaax_iova_table_p = NULL;
+
+ DPAAX_DEBUG("IOVA Table cleanedup");
+}
+
+int
+dpaax_iova_table_update(phys_addr_t paddr, void *vaddr, size_t length)
+{
+ int found = 0;
+ unsigned int i;
+ size_t req_length = length, e_offset;
+ struct dpaax_iovat_element *entry;
+ uintptr_t align_vaddr;
+ phys_addr_t align_paddr;
+
+ if (unlikely(dpaax_iova_table_p == NULL))
+ return -1;
+
+ align_paddr = paddr & DPAAX_MEM_SPLIT_MASK;
+ align_vaddr = ((uintptr_t)vaddr & DPAAX_MEM_SPLIT_MASK);
+
+ /* Check if paddr is available in table */
+ entry = dpaax_iova_table_p->entries;
+ for (i = 0; i < dpaax_iova_table_p->count; i++) {
+ if (align_paddr < entry[i].start) {
+ /* Address lower than start, but not found in previous
+ * iteration shouldn't exist.
+ */
+ DPAAX_ERR("Add: Incorrect entry for PA->VA Table"
+ "(%"PRIu64")", paddr);
+ DPAAX_ERR("Add: Lowest address: %"PRIu64"",
+ entry[i].start);
+ return -1;
+ }
+
+ if (align_paddr > (entry[i].start + entry[i].len))
+ continue;
+
+ /* align_paddr >= start && align_paddr < (start + len) */
+ found = 1;
+
+ do {
+ e_offset = ((align_paddr - entry[i].start) / DPAAX_MEM_SPLIT);
+ /* TODO: Whatif something already exists at this
+ * location - is that an error? For now, ignoring the
+ * case.
+ */
+ entry[i].pages[e_offset] = align_vaddr;
+ DPAAX_DEBUG("Added: vaddr=%zu for Phy:%"PRIu64" at %zu"
+ " remaining len %zu", align_vaddr,
+ align_paddr, e_offset, req_length);
+
+ /* Incoming request can be larger than the
+ * DPAAX_MEM_SPLIT size - in which case, multiple
+ * entries in entry->pages[] are filled up.
+ */
+ if (req_length <= DPAAX_MEM_SPLIT)
+ break;
+ align_paddr += DPAAX_MEM_SPLIT;
+ align_vaddr += DPAAX_MEM_SPLIT;
+ req_length -= DPAAX_MEM_SPLIT;
+ } while (1);
+
+ break;
+ }
+
+ if (!found) {
+ /* There might be case where the incoming physical address is
+ * beyond the address discovered in the memory node of
+ * device-tree. Specially if some malloc'd area is used by EAL
+ * and the memevent handlers passes that across. But, this is
+ * not necessarily an error.
+ */
+ DPAAX_DEBUG("Add: Unable to find slot for vaddr:(%p),"
+ " phy(%"PRIu64")",
+ vaddr, paddr);
+ return -1;
+ }
+
+ DPAAX_DEBUG("Add: Found slot at (%"PRIu64")[(%zu)] for vaddr:(%p),"
+ " phy(%"PRIu64"), len(%zu)", entry[i].start, e_offset,
+ vaddr, paddr, length);
+ return 0;
+}
+
+/* dpaax_iova_table_dump
+ * Dump the table, with its entries, on screen. Only works in Debug Mode
+ * Not for weak hearted - the tables can get quite large
+ */
+void
+dpaax_iova_table_dump(void)
+{
+ unsigned int i, j;
+ struct dpaax_iovat_element *entry;
+
+ /* In case DEBUG is not enabled, some 'if' conditions might misbehave
+ * as they have nothing else in them except a DPAAX_DEBUG() which if
+ * tuned out would leave 'if' naked.
+ */
+ if (rte_log_get_global_level() < RTE_LOG_DEBUG) {
+ DPAAX_ERR("Set log level to Debug for PA->Table dump!");
+ return;
+ }
+
+ DPAAX_DEBUG(" === Start of PA->VA Translation Table ===");
+ if (dpaax_iova_table_p == NULL)
+ DPAAX_DEBUG("\tNULL");
+
+ entry = dpaax_iova_table_p->entries;
+ for (i = 0; i < dpaax_iova_table_p->count; i++) {
+ DPAAX_DEBUG("\t(%16i),(%16"PRIu64"),(%16zu),(%16p)",
+ i, entry[i].start, entry[i].len, entry[i].pages);
+ DPAAX_DEBUG("\t\t (PA), (VA)");
+ for (j = 0; j < (entry->len/DPAAX_MEM_SPLIT); j++) {
+ if (entry[i].pages[j] == 0)
+ continue;
+ DPAAX_DEBUG("\t\t(%16"PRIx64"),(%16"PRIx64")",
+ (entry[i].start + (j * sizeof(uint64_t))),
+ entry[i].pages[j]);
+ }
+ }
+ DPAAX_DEBUG(" === End of PA->VA Translation Table ===");
+}
+
+static void
+dpaax_memevent_cb(enum rte_mem_event type, const void *addr, size_t len,
+ void *arg __rte_unused)
+{
+ struct rte_memseg_list *msl;
+ struct rte_memseg *ms;
+ size_t cur_len = 0, map_len = 0;
+ phys_addr_t phys_addr;
+ void *virt_addr;
+ int ret;
+
+ DPAAX_DEBUG("Called with addr=%p, len=%zu", addr, len);
+
+ msl = rte_mem_virt2memseg_list(addr);
+
+ while (cur_len < len) {
+ const void *va = RTE_PTR_ADD(addr, cur_len);
+
+ ms = rte_mem_virt2memseg(va, msl);
+ phys_addr = rte_mem_virt2phy(ms->addr);
+ virt_addr = ms->addr;
+ map_len = ms->len;
+
+ DPAAX_DEBUG("Request for %s, va=%p, virt_addr=%p,"
+ "iova=%"PRIu64", map_len=%zu",
+ type == RTE_MEM_EVENT_ALLOC ?
+ "alloc" : "dealloc",
+ va, virt_addr, phys_addr, map_len);
+
+ if (type == RTE_MEM_EVENT_ALLOC)
+ ret = dpaax_iova_table_update(phys_addr, virt_addr,
+ map_len);
+ else
+ /* In case of mem_events for MEM_EVENT_FREE, complete
+ * hugepage is released and its PA entry is set to 0.
+ */
+ ret = dpaax_iova_table_update(phys_addr, 0, map_len);
+
+ if (ret != 0) {
+ DPAAX_DEBUG("PA-Table entry update failed. "
+ "Map=%d, addr=%p, len=%zu, err:(%d)",
+ type, va, map_len, ret);
+ return;
+ }
+
+ cur_len += map_len;
+ }
+}
+
+static int
+dpaax_memevent_walk_memsegs(const struct rte_memseg_list *msl __rte_unused,
+ const struct rte_memseg *ms, size_t len,
+ void *arg __rte_unused)
+{
+ DPAAX_DEBUG("Walking for %p (pa=%"PRIu64") and len %zu",
+ ms->addr, ms->phys_addr, len);
+ dpaax_iova_table_update(rte_mem_virt2phy(ms->addr), ms->addr, len);
+ return 0;
+}
+
+static int
+dpaax_handle_memevents(void)
+{
+ /* First, walk through all memsegs and pin them, before installing
+ * handler. This assures that all memseg which have already been
+ * identified/allocated by EAL, are already part of PA->VA Table. This
+ * is especially for cases where application allocates memory before
+ * the EAL or this is an externally allocated memory passed to EAL.
+ */
+ rte_memseg_contig_walk_thread_unsafe(dpaax_memevent_walk_memsegs, NULL);
+
+ return rte_mem_event_callback_register("dpaax_memevents_cb",
+ dpaax_memevent_cb, NULL);
+}
+
+RTE_INIT(dpaax_log)
+{
+ dpaax_logger = rte_log_register("pmd.common.dpaax");
+ if (dpaax_logger >= 0)
+ rte_log_set_level(dpaax_logger, RTE_LOG_ERR);
+}
diff --git a/drivers/common/dpaax/dpaax_iova_table.h b/drivers/common/dpaax/dpaax_iova_table.h
new file mode 100644
index 00000000..138827e7
--- /dev/null
+++ b/drivers/common/dpaax/dpaax_iova_table.h
@@ -0,0 +1,105 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#ifndef _DPAAX_IOVA_TABLE_H_
+#define _DPAAX_IOVA_TABLE_H_
+
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <inttypes.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <glob.h>
+#include <errno.h>
+#include <arpa/inet.h>
+
+#include <rte_eal.h>
+#include <rte_branch_prediction.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+
+struct dpaax_iovat_element {
+ phys_addr_t start; /**< Start address of block of physical pages */
+ size_t len; /**< Difference of end-start for quick access */
+ uint64_t *pages; /**< VA for each physical page in this block */
+};
+
+struct dpaax_iova_table {
+ unsigned int count; /**< No. of blocks of contiguous physical pages */
+ struct dpaax_iovat_element entries[0];
+};
+
+/* Pointer to the table, which is common for DPAA/DPAA2 and only a single
+ * instance is required across net/crypto/event drivers. This table is
+ * populated iff devices are found on the bus.
+ */
+extern struct dpaax_iova_table *dpaax_iova_table_p;
+
+/* Device tree file for memory layout is named 'memory@<addr>' where the 'addr'
+ * is SoC dependent, or even Uboot fixup dependent.
+ */
+#define MEM_NODE_PATH_GLOB "/proc/device-tree/memory[@0-9]*/reg"
+/* Device file should be multiple of 16 bytes, each containing 8 byte of addr
+ * and its length. Assuming max of 5 entries.
+ */
+#define MEM_NODE_FILE_LEN ((16 * 5) + 1)
+
+/* Table is made up of DPAAX_MEM_SPLIT elements for each contiguous zone. This
+ * helps avoid separate handling for cases where more than one size of hugepage
+ * is supported.
+ */
+#define DPAAX_MEM_SPLIT (1<<21)
+#define DPAAX_MEM_SPLIT_MASK ~(DPAAX_MEM_SPLIT - 1) /**< Floor aligned */
+#define DPAAX_MEM_SPLIT_MASK_OFF (DPAAX_MEM_SPLIT - 1) /**< Offset */
+
+/* APIs exposed */
+int dpaax_iova_table_populate(void);
+void dpaax_iova_table_depopulate(void);
+int dpaax_iova_table_update(phys_addr_t paddr, void *vaddr, size_t length);
+void dpaax_iova_table_dump(void);
+
+static inline void *dpaax_iova_table_get_va(phys_addr_t paddr) __attribute__((hot));
+
+static inline void *
+dpaax_iova_table_get_va(phys_addr_t paddr) {
+ unsigned int i = 0, index;
+ void *vaddr = 0;
+ phys_addr_t paddr_align = paddr & DPAAX_MEM_SPLIT_MASK;
+ size_t offset = paddr & DPAAX_MEM_SPLIT_MASK_OFF;
+ struct dpaax_iovat_element *entry;
+
+ if (unlikely(dpaax_iova_table_p == NULL))
+ return NULL;
+
+ entry = dpaax_iova_table_p->entries;
+
+ do {
+ if (unlikely(i > dpaax_iova_table_p->count))
+ break;
+
+ if (paddr_align < entry[i].start) {
+ /* Incorrect paddr; Not in memory range */
+ return NULL;
+ }
+
+ if (paddr_align > (entry[i].start + entry[i].len)) {
+ i++;
+ continue;
+ }
+
+ /* paddr > entry->start && paddr <= entry->(start+len) */
+ index = (paddr_align - entry[i].start)/DPAAX_MEM_SPLIT;
+ vaddr = (void *)((uintptr_t)entry[i].pages[index] + offset);
+ break;
+ } while (1);
+
+ return vaddr;
+}
+
+#endif /* _DPAAX_IOVA_TABLE_H_ */
diff --git a/drivers/common/dpaax/dpaax_logs.h b/drivers/common/dpaax/dpaax_logs.h
new file mode 100644
index 00000000..bf1b27cc
--- /dev/null
+++ b/drivers/common/dpaax/dpaax_logs.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#ifndef _DPAAX_LOGS_H_
+#define _DPAAX_LOGS_H_
+
+#include <rte_log.h>
+
+extern int dpaax_logger;
+
+#define DPAAX_LOG(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, dpaax_logger, "dpaax: " fmt "\n", \
+ ##args)
+
+/* Debug logs are with Function names */
+#define DPAAX_DEBUG(fmt, args...) \
+ rte_log(RTE_LOG_DEBUG, dpaax_logger, "dpaax: %s(): " fmt "\n", \
+ __func__, ##args)
+
+#define DPAAX_INFO(fmt, args...) \
+ DPAAX_LOG(INFO, fmt, ## args)
+#define DPAAX_ERR(fmt, args...) \
+ DPAAX_LOG(ERR, fmt, ## args)
+#define DPAAX_WARN(fmt, args...) \
+ DPAAX_LOG(WARNING, fmt, ## args)
+
+/* DP Logs, toggled out at compile time if level lower than current level */
+#define DPAAX_DP_LOG(level, fmt, args...) \
+ RTE_LOG_DP(level, PMD, fmt, ## args)
+
+#define DPAAX_DP_DEBUG(fmt, args...) \
+ DPAAX_DP_LOG(DEBUG, fmt, ## args)
+#define DPAAX_DP_INFO(fmt, args...) \
+ DPAAX_DP_LOG(INFO, fmt, ## args)
+#define DPAAX_DP_WARN(fmt, args...) \
+ DPAAX_DP_LOG(WARNING, fmt, ## args)
+
+#endif /* _DPAAX_LOGS_H_ */
diff --git a/drivers/common/dpaax/meson.build b/drivers/common/dpaax/meson.build
new file mode 100644
index 00000000..98a1bdd4
--- /dev/null
+++ b/drivers/common/dpaax/meson.build
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 NXP
+
+allow_experimental_apis = true
+
+if host_machine.system() != 'linux'
+ build = false
+endif
+
+sources = files('dpaax_iova_table.c')
+
+cflags += ['-D_GNU_SOURCE']
diff --git a/drivers/common/dpaax/rte_common_dpaax_version.map b/drivers/common/dpaax/rte_common_dpaax_version.map
new file mode 100644
index 00000000..8131c9e3
--- /dev/null
+++ b/drivers/common/dpaax/rte_common_dpaax_version.map
@@ -0,0 +1,11 @@
+DPDK_18.11 {
+ global:
+
+ dpaax_iova_table_update;
+ dpaax_iova_table_depopulate;
+ dpaax_iova_table_dump;
+ dpaax_iova_table_p;
+ dpaax_iova_table_populate;
+
+ local: *;
+};
diff --git a/drivers/common/meson.build b/drivers/common/meson.build
index d7b7d8cf..a5093410 100644
--- a/drivers/common/meson.build
+++ b/drivers/common/meson.build
@@ -2,6 +2,6 @@
# Copyright(c) 2018 Cavium, Inc
std_deps = ['eal']
-drivers = ['octeontx', 'qat']
+drivers = ['cpt', 'dpaax', 'mvep', 'octeontx', 'qat']
config_flag_fmt = 'RTE_LIBRTE_@0@_COMMON'
driver_name_fmt = 'rte_common_@0@'
diff --git a/drivers/common/mvep/Makefile b/drivers/common/mvep/Makefile
new file mode 100644
index 00000000..1f5f005d
--- /dev/null
+++ b/drivers/common/mvep/Makefile
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Marvell International Ltd.
+#
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(MAKECMDGOALS),config)
+ifeq ($(LIBMUSDK_PATH),)
+$(error "Please define LIBMUSDK_PATH environment variable")
+endif
+endif
+endif
+
+# library name
+LIB = librte_common_mvep.a
+
+# library version
+LIBABIVER := 1
+
+# versioning export map
+EXPORT_MAP := rte_common_mvep_version.map
+
+# external library dependencies
+CFLAGS += -I$($RTE_SDK)/drivers/common/mvep
+CFLAGS += -I$(LIBMUSDK_PATH)/include
+CFLAGS += -DMVCONF_TYPES_PUBLIC
+CFLAGS += -DMVCONF_DMA_PHYS_ADDR_T_PUBLIC
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -O3
+LDLIBS += -L$(LIBMUSDK_PATH)/lib
+LDLIBS += -lmusdk
+LDLIBS += -lrte_eal -lrte_kvargs
+
+# library source files
+SRCS-y += mvep_common.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/common/mvep/meson.build b/drivers/common/mvep/meson.build
new file mode 100644
index 00000000..8ccfacb3
--- /dev/null
+++ b/drivers/common/mvep/meson.build
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Marvell International Ltd.
+# Copyright(c) 2018 Semihalf.
+# All rights reserved.
+#
+path = get_option('lib_musdk_dir')
+lib_dir = path + '/lib'
+inc_dir = path + '/include'
+
+lib = cc.find_library('libmusdk', dirs: [lib_dir], required: false)
+if not lib.found()
+ build = false
+else
+ ext_deps += lib
+ includes += include_directories(inc_dir)
+ cflags += ['-DMVCONF_TYPES_PUBLIC', '-DMVCONF_DMA_PHYS_ADDR_T_PUBLIC']
+endif
+
+sources = files('mvep_common.c')
diff --git a/drivers/common/mvep/mvep_common.c b/drivers/common/mvep/mvep_common.c
new file mode 100644
index 00000000..67fa65b5
--- /dev/null
+++ b/drivers/common/mvep/mvep_common.c
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ */
+
+#include <rte_common.h>
+
+#include <env/mv_autogen_comp_flags.h>
+#include <env/mv_sys_dma.h>
+
+#include "rte_mvep_common.h"
+
+/* Memory size (in bytes) for MUSDK dma buffers */
+#define MRVL_MUSDK_DMA_MEMSIZE (40 * 1024 * 1024)
+
+struct mvep {
+ uint32_t ref_count;
+};
+
+static struct mvep mvep;
+
+int rte_mvep_init(enum mvep_module_type module __rte_unused,
+ struct rte_kvargs *kvlist __rte_unused)
+{
+ int ret;
+
+ if (!mvep.ref_count) {
+ ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
+ if (ret)
+ return ret;
+ }
+
+ mvep.ref_count++;
+
+ return 0;
+}
+
+int rte_mvep_deinit(enum mvep_module_type module __rte_unused)
+{
+ mvep.ref_count--;
+
+ if (!mvep.ref_count)
+ mv_sys_dma_mem_destroy();
+
+ return 0;
+}
diff --git a/drivers/common/mvep/rte_common_mvep_version.map b/drivers/common/mvep/rte_common_mvep_version.map
new file mode 100644
index 00000000..c71722d7
--- /dev/null
+++ b/drivers/common/mvep/rte_common_mvep_version.map
@@ -0,0 +1,6 @@
+DPDK_18.11 {
+ global:
+
+ rte_mvep_init;
+ rte_mvep_deinit;
+};
diff --git a/drivers/common/mvep/rte_mvep_common.h b/drivers/common/mvep/rte_mvep_common.h
new file mode 100644
index 00000000..0593cefc
--- /dev/null
+++ b/drivers/common/mvep/rte_mvep_common.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ */
+
+#ifndef __RTE_MVEP_COMMON_H__
+#define __RTE_MVEP_COMMON_H__
+
+#include <rte_kvargs.h>
+
+enum mvep_module_type {
+ MVEP_MOD_T_NONE = 0,
+ MVEP_MOD_T_PP2,
+ MVEP_MOD_T_SAM,
+ MVEP_MOD_T_NETA,
+ MVEP_MOD_T_LAST
+};
+
+int rte_mvep_init(enum mvep_module_type module, struct rte_kvargs *kvlist);
+int rte_mvep_deinit(enum mvep_module_type module);
+
+#endif /* __RTE_MVEP_COMMON_H__ */
diff --git a/drivers/common/qat/qat_common.h b/drivers/common/qat/qat_common.h
index d4bef539..de9a3ba5 100644
--- a/drivers/common/qat/qat_common.h
+++ b/drivers/common/qat/qat_common.h
@@ -17,7 +17,8 @@
*/
enum qat_device_gen {
QAT_GEN1 = 1,
- QAT_GEN2
+ QAT_GEN2,
+ QAT_GEN3
};
enum qat_service_type {
diff --git a/drivers/common/qat/qat_device.c b/drivers/common/qat/qat_device.c
index f32d7235..2a1cf3e1 100644
--- a/drivers/common/qat/qat_device.c
+++ b/drivers/common/qat/qat_device.c
@@ -7,6 +7,7 @@
#include "qat_device.h"
#include "adf_transport_access_macros.h"
#include "qat_sym_pmd.h"
+#include "qat_comp_pmd.h"
/* Hardware device information per generation */
__extension__
@@ -14,11 +15,18 @@ struct qat_gen_hw_data qat_gen_config[] = {
[QAT_GEN1] = {
.dev_gen = QAT_GEN1,
.qp_hw_data = qat_gen1_qps,
+ .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1
},
[QAT_GEN2] = {
.dev_gen = QAT_GEN2,
.qp_hw_data = qat_gen1_qps,
/* gen2 has same ring layout as gen1 */
+ .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2
+ },
+ [QAT_GEN3] = {
+ .dev_gen = QAT_GEN3,
+ .qp_hw_data = qat_gen3_qps,
+ .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
},
};
@@ -43,10 +51,12 @@ static const struct rte_pci_id pci_id_qat_map[] = {
{
RTE_PCI_DEVICE(0x8086, 0x6f55),
},
+ {
+ RTE_PCI_DEVICE(0x8086, 0x18a1),
+ },
{.device_id = 0},
};
-
static struct qat_pci_device *
qat_pci_get_dev(uint8_t dev_id)
{
@@ -130,6 +140,9 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev)
case 0x6f55:
qat_dev->qat_dev_gen = QAT_GEN2;
break;
+ case 0x18a1:
+ qat_dev->qat_dev_gen = QAT_GEN3;
+ break;
default:
QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
return NULL;
@@ -187,6 +200,7 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
struct rte_pci_device *pci_dev)
{
int ret = 0;
+ int num_pmds_created = 0;
struct qat_pci_device *qat_pci_dev;
QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
@@ -199,23 +213,33 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
return -ENODEV;
ret = qat_sym_dev_create(qat_pci_dev);
- if (ret != 0)
- goto error_out;
+ if (ret == 0)
+ num_pmds_created++;
+ else
+ QAT_LOG(WARNING,
+ "Failed to create QAT SYM PMD on device %s",
+ qat_pci_dev->name);
ret = qat_comp_dev_create(qat_pci_dev);
- if (ret != 0)
- goto error_out;
+ if (ret == 0)
+ num_pmds_created++;
+ else
+ QAT_LOG(WARNING,
+ "Failed to create QAT COMP PMD on device %s",
+ qat_pci_dev->name);
ret = qat_asym_dev_create(qat_pci_dev);
- if (ret != 0)
- goto error_out;
-
- return 0;
+ if (ret == 0)
+ num_pmds_created++;
+ else
+ QAT_LOG(WARNING,
+ "Failed to create QAT ASYM PMD on device %s",
+ qat_pci_dev->name);
-error_out:
- qat_pci_dev_destroy(qat_pci_dev, pci_dev);
- return ret;
+ if (num_pmds_created == 0)
+ qat_pci_dev_destroy(qat_pci_dev, pci_dev);
+ return 0;
}
static int qat_pci_remove(struct rte_pci_device *pci_dev)
@@ -239,37 +263,37 @@ static struct rte_pci_driver rte_qat_pmd = {
.remove = qat_pci_remove
};
-__attribute__((weak)) int
+__rte_weak int
qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
{
return 0;
}
-__attribute__((weak)) int
+__rte_weak int
qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
{
return 0;
}
-__attribute__((weak)) int
+__rte_weak int
qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
{
return 0;
}
-__attribute__((weak)) int
+__rte_weak int
qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
{
return 0;
}
-__attribute__((weak)) int
+__rte_weak int
qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
{
return 0;
}
-__attribute__((weak)) int
+__rte_weak int
qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
{
return 0;
diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h
index 9599fc59..eb81c78f 100644
--- a/drivers/common/qat/qat_device.h
+++ b/drivers/common/qat/qat_device.h
@@ -16,6 +16,12 @@
#define QAT_DEV_NAME_MAX_LEN 64
+enum qat_comp_num_im_buffers {
+ QAT_NUM_INTERM_BUFS_GEN1 = 12,
+ QAT_NUM_INTERM_BUFS_GEN2 = 20,
+ QAT_NUM_INTERM_BUFS_GEN3 = 20
+};
+
/*
* This struct holds all the data about a QAT pci device
* including data about all services it supports.
@@ -59,6 +65,11 @@ struct qat_pci_device {
/* Data relating to compression service */
struct qat_comp_dev_private *comp_dev;
/**< link back to compressdev private data */
+ struct rte_device comp_rte_dev;
+ /**< This represents the compression subset of this pci device.
+ * Register with this rather than with the one in
+ * pci_dev so that its driver can have a compression-specific name
+ */
/* Data relating to asymmetric crypto service */
@@ -67,6 +78,7 @@ struct qat_pci_device {
struct qat_gen_hw_data {
enum qat_device_gen dev_gen;
const struct qat_qp_hw_data (*qp_hw_data)[ADF_MAX_QPS_ON_ANY_SERVICE];
+ enum qat_comp_num_im_buffers comp_num_im_bufs_required;
};
extern struct qat_gen_hw_data qat_gen_config[];
diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c
index 7ca7a45e..79f6a01b 100644
--- a/drivers/common/qat/qat_qp.c
+++ b/drivers/common/qat/qat_qp.c
@@ -90,6 +90,44 @@ const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
}
};
+__extension__
+const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
+ [ADF_MAX_QPS_ON_ANY_SERVICE] = {
+ /* queue pairs which provide an asymmetric crypto service */
+ [QAT_SERVICE_ASYMMETRIC] = {
+ {
+ .service_type = QAT_SERVICE_ASYMMETRIC,
+ .hw_bundle_num = 0,
+ .tx_ring_num = 0,
+ .rx_ring_num = 4,
+ .tx_msg_size = 64,
+ .rx_msg_size = 32,
+ }
+ },
+ /* queue pairs which provide a symmetric crypto service */
+ [QAT_SERVICE_SYMMETRIC] = {
+ {
+ .service_type = QAT_SERVICE_SYMMETRIC,
+ .hw_bundle_num = 0,
+ .tx_ring_num = 1,
+ .rx_ring_num = 5,
+ .tx_msg_size = 128,
+ .rx_msg_size = 32,
+ }
+ },
+ /* queue pairs which provide a compression service */
+ [QAT_SERVICE_COMPRESSION] = {
+ {
+ .service_type = QAT_SERVICE_COMPRESSION,
+ .hw_bundle_num = 0,
+ .tx_ring_num = 3,
+ .rx_ring_num = 7,
+ .tx_msg_size = 128,
+ .rx_msg_size = 32,
+ }
+ }
+};
+
static int qat_qp_check_queue_alignment(uint64_t phys_addr,
uint32_t queue_size_bytes);
static void qat_queue_delete(struct qat_queue *queue);
@@ -596,15 +634,23 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
uint32_t head;
uint32_t resp_counter = 0;
uint8_t *resp_msg;
+ uint8_t hdr_flags;
rx_queue = &(tmp_qp->rx_q);
tx_queue = &(tmp_qp->tx_q);
head = rx_queue->head;
resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
+ hdr_flags = ((struct icp_qat_fw_comn_resp_hdr *)resp_msg)->hdr_flags;
while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
resp_counter != nb_ops) {
+ if (unlikely(!ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_flags))) {
+ /* Fatal firmware error */
+ QAT_LOG(ERR, "QAT Firmware returned invalid response");
+ return 0;
+ }
+
if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
qat_sym_process_response(ops, resp_msg);
else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
@@ -635,7 +681,7 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
return resp_counter;
}
-__attribute__((weak)) int
+__rte_weak int
qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused)
{
return 0;
diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h
index 69f8a613..6f1525e1 100644
--- a/drivers/common/qat/qat_qp.h
+++ b/drivers/common/qat/qat_qp.h
@@ -85,6 +85,7 @@ struct qat_qp {
} __rte_cache_aligned;
extern const struct qat_qp_hw_data qat_gen1_qps[][ADF_MAX_QPS_ON_ANY_SERVICE];
+extern const struct qat_qp_hw_data qat_gen3_qps[][ADF_MAX_QPS_ON_ANY_SERVICE];
uint16_t
qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops);